From c15593197453b8bf90fc3a9080ba2afeaf7934ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=B6hrmann?= Date: Sat, 21 Nov 2020 17:44:04 +0100 Subject: [PATCH] internal/cpu: add darwin/arm64 CPU feature detection support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #42747 Change-Id: I6b1679348c77161f075f0678818bb003fc0e8c86 Reviewed-on: https://go-review.googlesource.com/c/go/+/271989 Trust: Martin Möhrmann Run-TryBot: Martin Möhrmann TryBot-Result: Go Bot Reviewed-by: Cherry Zhang --- src/internal/cpu/cpu_android.go | 7 -- src/internal/cpu/cpu_arm64.go | 97 +------------------ .../{cpu_linux.go => cpu_arm64_android.go} | 6 +- src/internal/cpu/cpu_arm64_darwin.go | 34 +++++++ src/internal/cpu/cpu_arm64_freebsd.go | 45 +++++++++ src/internal/cpu/cpu_arm64_hwcap.go | 63 ++++++++++++ .../cpu/{cpu_other.go => cpu_arm64_linux.go} | 8 +- src/internal/cpu/cpu_arm64_other.go | 17 ++++ src/internal/cpu/cpu_freebsd.go | 7 -- src/internal/cpu/cpu_test.go | 7 +- src/runtime/os_darwin.go | 12 +++ src/runtime/sys_darwin.go | 10 +- src/runtime/sys_darwin_amd64.s | 20 +++- src/runtime/sys_darwin_arm64.s | 18 +++- 14 files changed, 223 insertions(+), 128 deletions(-) delete mode 100644 src/internal/cpu/cpu_android.go rename src/internal/cpu/{cpu_linux.go => cpu_arm64_android.go} (75%) create mode 100644 src/internal/cpu/cpu_arm64_darwin.go create mode 100644 src/internal/cpu/cpu_arm64_freebsd.go create mode 100644 src/internal/cpu/cpu_arm64_hwcap.go rename src/internal/cpu/{cpu_other.go => cpu_arm64_linux.go} (73%) create mode 100644 src/internal/cpu/cpu_arm64_other.go delete mode 100644 src/internal/cpu/cpu_freebsd.go diff --git a/src/internal/cpu/cpu_android.go b/src/internal/cpu/cpu_android.go deleted file mode 100644 index d995e8d5a74055..00000000000000 --- a/src/internal/cpu/cpu_android.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cpu - -const GOOS = "android" diff --git a/src/internal/cpu/cpu_arm64.go b/src/internal/cpu/cpu_arm64.go index a8f7b2b458bac9..f64d9e4dd3100a 100644 --- a/src/internal/cpu/cpu_arm64.go +++ b/src/internal/cpu/cpu_arm64.go @@ -6,21 +6,6 @@ package cpu const CacheLinePadSize = 64 -// HWCap may be initialized by archauxv and -// should not be changed after it was initialized. -var HWCap uint - -// HWCAP bits. These are exposed by Linux. -const ( - hwcap_AES = 1 << 3 - hwcap_PMULL = 1 << 4 - hwcap_SHA1 = 1 << 5 - hwcap_SHA2 = 1 << 6 - hwcap_CRC32 = 1 << 7 - hwcap_ATOMICS = 1 << 8 - hwcap_CPUID = 1 << 11 -) - func doinit() { options = []option{ {Name: "aes", Feature: &ARM64.HasAES}, @@ -34,86 +19,8 @@ func doinit() { {Name: "isZeus", Feature: &ARM64.IsZeus}, } - switch GOOS { - case "linux", "android": - // HWCap was populated by the runtime from the auxiliary vector. - // Use HWCap information since reading aarch64 system registers - // is not supported in user space on older linux kernels. - ARM64.HasAES = isSet(HWCap, hwcap_AES) - ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL) - ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1) - ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2) - ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32) - ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID) - - // The Samsung S9+ kernel reports support for atomics, but not all cores - // actually support them, resulting in SIGILL. See issue #28431. - // TODO(elias.naur): Only disable the optimization on bad chipsets on android. - ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && GOOS != "android" - - // Check to see if executing on a NeoverseN1 and in order to do that, - // check the AUXV for the CPUID bit. The getMIDR function executes an - // instruction which would normally be an illegal instruction, but it's - // trapped by the kernel, the value sanitized and then returned. Without - // the CPUID bit the kernel will not trap the instruction and the process - // will be terminated with SIGILL. - if ARM64.HasCPUID { - midr := getMIDR() - part_num := uint16((midr >> 4) & 0xfff) - implementor := byte((midr >> 24) & 0xff) - - if implementor == 'A' && part_num == 0xd0c { - ARM64.IsNeoverseN1 = true - } - if implementor == 'A' && part_num == 0xd40 { - ARM64.IsZeus = true - } - } - - case "freebsd": - // Retrieve info from system register ID_AA64ISAR0_EL1. - isar0 := getisar0() - - // ID_AA64ISAR0_EL1 - switch extractBits(isar0, 4, 7) { - case 1: - ARM64.HasAES = true - case 2: - ARM64.HasAES = true - ARM64.HasPMULL = true - } - - switch extractBits(isar0, 8, 11) { - case 1: - ARM64.HasSHA1 = true - } - - switch extractBits(isar0, 12, 15) { - case 1, 2: - ARM64.HasSHA2 = true - } - - switch extractBits(isar0, 16, 19) { - case 1: - ARM64.HasCRC32 = true - } - - switch extractBits(isar0, 20, 23) { - case 2: - ARM64.HasATOMICS = true - } - default: - // Other operating systems do not support reading HWCap from auxiliary vector - // or reading privileged aarch64 system registers in user space. - } -} - -func extractBits(data uint64, start, end uint) uint { - return (uint)(data>>start) & ((1 << (end - start + 1)) - 1) -} - -func isSet(hwc uint, value uint) bool { - return hwc&value != 0 + // arm64 uses different ways to detect CPU features at runtime depending on the operating system. + osInit() } func getisar0() uint64 diff --git a/src/internal/cpu/cpu_linux.go b/src/internal/cpu/cpu_arm64_android.go similarity index 75% rename from src/internal/cpu/cpu_linux.go rename to src/internal/cpu/cpu_arm64_android.go index ec0b84c510f7c8..3c9e57c52aa14c 100644 --- a/src/internal/cpu/cpu_linux.go +++ b/src/internal/cpu/cpu_arm64_android.go @@ -2,8 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !android +// +build arm64 package cpu -const GOOS = "linux" +func osInit() { + hwcapInit("android") +} diff --git a/src/internal/cpu/cpu_arm64_darwin.go b/src/internal/cpu/cpu_arm64_darwin.go new file mode 100644 index 00000000000000..e094b97f973c39 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_darwin.go @@ -0,0 +1,34 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 +// +build darwin +// +build !ios + +package cpu + +func osInit() { + ARM64.HasATOMICS = sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00")) + ARM64.HasCRC32 = sysctlEnabled([]byte("hw.optional.armv8_crc32\x00")) + + // There are no hw.optional sysctl values for the below features on Mac OS 11.0 + // to detect their supported state dynamically. Assume the CPU features that + // Apple Silicon M1 supports to be available as a minimal set of features + // to all Go programs running on darwin/arm64. + ARM64.HasAES = true + ARM64.HasPMULL = true + ARM64.HasSHA1 = true + ARM64.HasSHA2 = true +} + +//go:noescape +func getsysctlbyname(name []byte) (int32, int32) + +func sysctlEnabled(name []byte) bool { + ret, value := getsysctlbyname(name) + if ret < 0 { + return false + } + return value > 0 +} diff --git a/src/internal/cpu/cpu_arm64_freebsd.go b/src/internal/cpu/cpu_arm64_freebsd.go new file mode 100644 index 00000000000000..9de2005c2e3684 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_freebsd.go @@ -0,0 +1,45 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 + +package cpu + +func osInit() { + // Retrieve info from system register ID_AA64ISAR0_EL1. + isar0 := getisar0() + + // ID_AA64ISAR0_EL1 + switch extractBits(isar0, 4, 7) { + case 1: + ARM64.HasAES = true + case 2: + ARM64.HasAES = true + ARM64.HasPMULL = true + } + + switch extractBits(isar0, 8, 11) { + case 1: + ARM64.HasSHA1 = true + } + + switch extractBits(isar0, 12, 15) { + case 1, 2: + ARM64.HasSHA2 = true + } + + switch extractBits(isar0, 16, 19) { + case 1: + ARM64.HasCRC32 = true + } + + switch extractBits(isar0, 20, 23) { + case 2: + ARM64.HasATOMICS = true + } +} + +func extractBits(data uint64, start, end uint) uint { + return (uint)(data>>start) & ((1 << (end - start + 1)) - 1) +} diff --git a/src/internal/cpu/cpu_arm64_hwcap.go b/src/internal/cpu/cpu_arm64_hwcap.go new file mode 100644 index 00000000000000..fdaf43e1a25e05 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_hwcap.go @@ -0,0 +1,63 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 +// +build linux + +package cpu + +// HWCap may be initialized by archauxv and +// should not be changed after it was initialized. +var HWCap uint + +// HWCAP bits. These are exposed by Linux. +const ( + hwcap_AES = 1 << 3 + hwcap_PMULL = 1 << 4 + hwcap_SHA1 = 1 << 5 + hwcap_SHA2 = 1 << 6 + hwcap_CRC32 = 1 << 7 + hwcap_ATOMICS = 1 << 8 + hwcap_CPUID = 1 << 11 +) + +func hwcapInit(os string) { + // HWCap was populated by the runtime from the auxiliary vector. + // Use HWCap information since reading aarch64 system registers + // is not supported in user space on older linux kernels. + ARM64.HasAES = isSet(HWCap, hwcap_AES) + ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL) + ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1) + ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2) + ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32) + ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID) + + // The Samsung S9+ kernel reports support for atomics, but not all cores + // actually support them, resulting in SIGILL. See issue #28431. + // TODO(elias.naur): Only disable the optimization on bad chipsets on android. + ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && os != "android" + + // Check to see if executing on a NeoverseN1 and in order to do that, + // check the AUXV for the CPUID bit. The getMIDR function executes an + // instruction which would normally be an illegal instruction, but it's + // trapped by the kernel, the value sanitized and then returned. Without + // the CPUID bit the kernel will not trap the instruction and the process + // will be terminated with SIGILL. + if ARM64.HasCPUID { + midr := getMIDR() + part_num := uint16((midr >> 4) & 0xfff) + implementor := byte((midr >> 24) & 0xff) + + if implementor == 'A' && part_num == 0xd0c { + ARM64.IsNeoverseN1 = true + } + if implementor == 'A' && part_num == 0xd40 { + ARM64.IsZeus = true + } + } +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/src/internal/cpu/cpu_other.go b/src/internal/cpu/cpu_arm64_linux.go similarity index 73% rename from src/internal/cpu/cpu_other.go rename to src/internal/cpu/cpu_arm64_linux.go index 8a15fbe79d8134..2f7411ff1e1199 100644 --- a/src/internal/cpu/cpu_other.go +++ b/src/internal/cpu/cpu_arm64_linux.go @@ -2,10 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !linux -// +build !freebsd +// +build arm64 +// +build linux // +build !android package cpu -const GOOS = "other" +func osInit() { + hwcapInit("linux") +} diff --git a/src/internal/cpu/cpu_arm64_other.go b/src/internal/cpu/cpu_arm64_other.go new file mode 100644 index 00000000000000..f191db28d2a0d3 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_other.go @@ -0,0 +1,17 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 +// +build !linux +// +build !freebsd +// +build !android +// +build !darwin ios + +package cpu + +func osInit() { + // Other operating systems do not support reading HWCap from auxiliary vector, + // reading privileged aarch64 system registers or sysctl in user space to detect + // CPU features at runtime. +} diff --git a/src/internal/cpu/cpu_freebsd.go b/src/internal/cpu/cpu_freebsd.go deleted file mode 100644 index dc37173dacf1c5..00000000000000 --- a/src/internal/cpu/cpu_freebsd.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cpu - -const GOOS = "freebsd" diff --git a/src/internal/cpu/cpu_test.go b/src/internal/cpu/cpu_test.go index 919bbd5ed72e41..2de7365732d748 100644 --- a/src/internal/cpu/cpu_test.go +++ b/src/internal/cpu/cpu_test.go @@ -18,7 +18,7 @@ func TestMinimalFeatures(t *testing.T) { // TODO: maybe do MustSupportFeatureDectection(t) ? if runtime.GOARCH == "arm64" { switch runtime.GOOS { - case "linux", "android": + case "linux", "android", "darwin": default: t.Skipf("%s/%s is not supported", runtime.GOOS, runtime.GOARCH) } @@ -38,10 +38,7 @@ func MustHaveDebugOptionsSupport(t *testing.T) { } func MustSupportFeatureDectection(t *testing.T) { - if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { - t.Skipf("CPU feature detection is not supported on %s/%s", runtime.GOOS, runtime.GOARCH) - } - // TODO: maybe there are other platforms? + // TODO: add platforms that do not have CPU feature detection support. } func runDebugOptionsTest(t *testing.T, test string, options string) { diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 52f3cd1fefb9cc..e0a43c28aaa693 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -130,6 +130,18 @@ func osinit() { physPageSize = getPageSize() } +func sysctlbynameInt32(name []byte) (int32, int32) { + out := int32(0) + nout := unsafe.Sizeof(out) + ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + return ret, out +} + +//go:linkname internal_cpu_getsysctlbyname internal/cpu.getsysctlbyname +func internal_cpu_getsysctlbyname(name []byte) (int32, int32) { + return sysctlbynameInt32(name) +} + const ( _CTL_HW = 6 _HW_NCPU = 3 diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index c63ba8c6cd27c4..c89ce780124cbe 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -360,11 +360,18 @@ func setitimer_trampoline() //go:nosplit //go:cgo_unsafe_args -func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { +func sysctl(mib *uint32, miblen uint32, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 { return libcCall(unsafe.Pointer(funcPC(sysctl_trampoline)), unsafe.Pointer(&mib)) } func sysctl_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func sysctlbyname(name *byte, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 { + return libcCall(unsafe.Pointer(funcPC(sysctlbyname_trampoline)), unsafe.Pointer(&name)) +} +func sysctlbyname_trampoline() + //go:nosplit //go:cgo_unsafe_args func fcntl(fd, cmd, arg int32) int32 { @@ -486,6 +493,7 @@ func setNonblock(fd int32) { //go:cgo_import_dynamic libc_kill kill "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_setitimer setitimer "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" +//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_fcntl fcntl "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_kqueue kqueue "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_kevent kevent "/usr/lib/libSystem.B.dylib" diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s index 9b5b23901dcd9e..630fb5df64fd2c 100644 --- a/src/runtime/sys_darwin_amd64.s +++ b/src/runtime/sys_darwin_amd64.s @@ -371,15 +371,27 @@ TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0 PUSHQ BP MOVQ SP, BP MOVL 8(DI), SI // arg 2 miblen - MOVQ 16(DI), DX // arg 3 out - MOVQ 24(DI), CX // arg 4 size - MOVQ 32(DI), R8 // arg 5 dst - MOVQ 40(DI), R9 // arg 6 ndst + MOVQ 16(DI), DX // arg 3 oldp + MOVQ 24(DI), CX // arg 4 oldlenp + MOVQ 32(DI), R8 // arg 5 newp + MOVQ 40(DI), R9 // arg 6 newlen MOVQ 0(DI), DI // arg 1 mib CALL libc_sysctl(SB) POPQ BP RET +TEXT runtime·sysctlbyname_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 oldp + MOVQ 16(DI), DX // arg 3 oldlenp + MOVQ 24(DI), CX // arg 4 newp + MOVQ 32(DI), R8 // arg 5 newlen + MOVQ 0(DI), DI // arg 1 name + CALL libc_sysctlbyname(SB) + POPQ BP + RET + TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0 PUSHQ BP MOVQ SP, BP diff --git a/src/runtime/sys_darwin_arm64.s b/src/runtime/sys_darwin_arm64.s index 9d4d116c507d09..96d2ed10767d85 100644 --- a/src/runtime/sys_darwin_arm64.s +++ b/src/runtime/sys_darwin_arm64.s @@ -301,14 +301,24 @@ TEXT runtime·usleep_trampoline(SB),NOSPLIT,$0 TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0 MOVW 8(R0), R1 // arg 2 miblen - MOVD 16(R0), R2 // arg 3 out - MOVD 24(R0), R3 // arg 4 size - MOVD 32(R0), R4 // arg 5 dst - MOVD 40(R0), R5 // arg 6 ndst + MOVD 16(R0), R2 // arg 3 oldp + MOVD 24(R0), R3 // arg 4 oldlenp + MOVD 32(R0), R4 // arg 5 newp + MOVD 40(R0), R5 // arg 6 newlen MOVD 0(R0), R0 // arg 1 mib BL libc_sysctl(SB) RET +TEXT runtime·sysctlbyname_trampoline(SB),NOSPLIT,$0 + MOVD 8(R0), R1 // arg 2 oldp + MOVD 16(R0), R2 // arg 3 oldlenp + MOVD 24(R0), R3 // arg 4 newp + MOVD 32(R0), R4 // arg 5 newlen + MOVD 0(R0), R0 // arg 1 name + BL libc_sysctlbyname(SB) + RET + + TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0 BL libc_kqueue(SB) RET