diff --git a/cpu/cpu.go b/cpu/cpu.go index abbec2d44b..c8df189daf 100644 --- a/cpu/cpu.go +++ b/cpu/cpu.go @@ -100,6 +100,7 @@ var ARM64 struct { HasSHA512 bool // SHA512 hardware implementation HasSVE bool // Scalable Vector Extensions HasASIMDFHM bool // Advanced SIMD multiplication FP16 to FP32 + HasFMA bool // Fused Multiply Add _ CacheLinePad } diff --git a/cpu/cpu_android_arm64.go b/cpu/cpu_android_arm64.go new file mode 100644 index 0000000000..a6dbb15688 --- /dev/null +++ b/cpu/cpu_android_arm64.go @@ -0,0 +1,78 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 +// +build android + +package cpu + +// HWCAP/HWCAP2 bits. These are exposed by Linux. +const ( + hwcap_FP = 1 << 0 + hwcap_ASIMD = 1 << 1 + hwcap_EVTSTRM = 1 << 2 + hwcap_AES = 1 << 3 + hwcap_PMULL = 1 << 4 + hwcap_SHA1 = 1 << 5 + hwcap_SHA2 = 1 << 6 + hwcap_CRC32 = 1 << 7 + hwcap_ATOMICS = 1 << 8 + hwcap_FPHP = 1 << 9 + hwcap_ASIMDHP = 1 << 10 + hwcap_CPUID = 1 << 11 + hwcap_ASIMDRDM = 1 << 12 + hwcap_JSCVT = 1 << 13 + hwcap_FCMA = 1 << 14 + hwcap_LRCPC = 1 << 15 + hwcap_DCPOP = 1 << 16 + hwcap_SHA3 = 1 << 17 + hwcap_SM3 = 1 << 18 + hwcap_SM4 = 1 << 19 + hwcap_ASIMDDP = 1 << 20 + hwcap_SHA512 = 1 << 21 + hwcap_SVE = 1 << 22 + hwcap_ASIMDFHM = 1 << 23 +) + +func osInit() { + if err := readHWCAP(); err != nil { + // failed to read /proc/self/auxv, try reading registers directly + readARM64Registers() + return + } + + // Use HWCap information since reading aarch64 system registers + // is not supported in user space on older linux kernels. + ARM64.HasFP = isSet(hwCap, hwcap_FP) + ARM64.HasASIMD = isSet(hwCap, hwcap_ASIMD) + ARM64.HasEVTSTRM = isSet(hwCap, hwcap_EVTSTRM) + ARM64.HasAES = isSet(hwCap, hwcap_AES) + ARM64.HasPMULL = isSet(hwCap, hwcap_PMULL) + ARM64.HasSHA1 = isSet(hwCap, hwcap_SHA1) + ARM64.HasSHA2 = isSet(hwCap, hwcap_SHA2) + ARM64.HasCRC32 = isSet(hwCap, hwcap_CRC32) + ARM64.HasFPHP = isSet(hwCap, hwcap_FPHP) + ARM64.HasASIMDHP = isSet(hwCap, hwcap_ASIMDHP) + ARM64.HasASIMDRDM = isSet(hwCap, hwcap_ASIMDRDM) + ARM64.HasJSCVT = isSet(hwCap, hwcap_JSCVT) + ARM64.HasFCMA = isSet(hwCap, hwcap_FCMA) + ARM64.HasLRCPC = isSet(hwCap, hwcap_LRCPC) + ARM64.HasDCPOP = isSet(hwCap, hwcap_DCPOP) + ARM64.HasSHA3 = isSet(hwCap, hwcap_SHA3) + ARM64.HasSM3 = isSet(hwCap, hwcap_SM3) + ARM64.HasSM4 = isSet(hwCap, hwcap_SM4) + ARM64.HasASIMDDP = isSet(hwCap, hwcap_ASIMDDP) + ARM64.HasSHA512 = isSet(hwCap, hwcap_SHA512) + ARM64.HasSVE = isSet(hwCap, hwcap_SVE) + ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM) + + // The Samsung S9+ kernel reports support for atomics, but not all cores + // actually support them, resulting in SIGILL. See issue #28431. + // TODO(elias.naur): Only disable the optimization on bad chipsets on android. + ARM64.HasATOMICS = false +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/cpu/cpu_arm64.go b/cpu/cpu_arm64.go index 87dd5e3021..c9d9dd03e4 100644 --- a/cpu/cpu_arm64.go +++ b/cpu/cpu_arm64.go @@ -41,8 +41,8 @@ func archInit() { switch runtime.GOOS { case "freebsd": readARM64Registers() - case "linux", "netbsd": - doinit() + case "linux", "netbsd", "android", "darwin": + osInit() default: // Most platforms don't seem to allow reading these registers. // @@ -52,13 +52,6 @@ func archInit() { } } -// setMinimalFeatures fakes the minimal ARM64 features expected by -// TestARM64minimalFeatures. -func setMinimalFeatures() { - ARM64.HasASIMD = true - ARM64.HasFP = true -} - func readARM64Registers() { Initialized = true @@ -170,3 +163,10 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) { func extractBits(data uint64, start, end uint) uint { return (uint)(data>>start) & ((1 << (end - start + 1)) - 1) } + +// setMinimalFeatures fakes the minimal ARM64 features expected by +// TestARM64minimalFeatures. +func setMinimalFeatures() { + ARM64.HasASIMD = true + ARM64.HasFP = true +} diff --git a/cpu/cpu_darwin_arm64.go b/cpu/cpu_darwin_arm64.go new file mode 100644 index 0000000000..be9124d9bf --- /dev/null +++ b/cpu/cpu_darwin_arm64.go @@ -0,0 +1,74 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 +// +build darwin +// +build !ios + +package cpu + +import ( + "unsafe" +) + +const ( + commpageHasNeonFP16 uint64 = 0x00000008 // ARM v8.2 NEON FP16 supported + commpageHasNeon uint64 = 0x00000100 // Advanced SIMD is supported + commpageHasNeonHPFP uint64 = 0x00000200 // Advanced SIMD half-precision + commpageHasVfp uint64 = 0x00000400 // VFP is supported + commpageHasEvent uint64 = 0x00001000 // WFE/SVE and period event wakeup + commpageHasFMA uint64 = 0x00002000 // Fused multiply add is supported + commpageHasARMv82FHM uint64 = 0x00004000 // Optional ARMv8.2 FMLAL/FMLSL instructions (required in ARMv8.4) + commpageHasARMv8Crypto uint64 = 0x01000000 // Optional ARMv8 Crypto extensions + commpageHasARMv81Atomics uint64 = 0x02000000 // ARMv8.1 Atomic instructions supported + commpageHasARMv8Crc32 uint64 = 0x04000000 // Optional ARMv8 crc32 instructions (required in ARMv8.1) + commpageHasARMv82SHA512 uint64 = 0x80000000 // Optional ARMv8.2 SHA512 instructions + commpageHasARMv82SHA3 uint64 = 0x0000000100000000 // Optional ARMv8.2 SHA3 instructions +) + +func osInit() { + ARM64.HasFP = darwinCheckFeatureEnabled(commpageHasVfp) + ARM64.HasASIMD = darwinCheckFeatureEnabled(commpageHasNeon) + ARM64.HasCRC32 = darwinCheckFeatureEnabled(commpageHasARMv8Crc32) + ARM64.HasATOMICS = darwinCheckFeatureEnabled(commpageHasARMv81Atomics) + ARM64.HasFPHP = darwinCheckFeatureEnabled(commpageHasNeonFP16) + ARM64.HasASIMDHP = darwinCheckFeatureEnabled(commpageHasNeonHPFP) + ARM64.HasSHA3 = darwinCheckFeatureEnabled(commpageHasARMv82SHA3) + ARM64.HasSHA512 = darwinCheckFeatureEnabled(commpageHasARMv82SHA512) + ARM64.HasASIMDFHM = darwinCheckFeatureEnabled(commpageHasARMv82FHM) + ARM64.HasSVE = darwinCheckFeatureEnabled(commpageHasEvent) + ARM64.HasFMA = darwinCheckFeatureEnabled(commpageHasFMA) + + // There are no hw.optional sysctl values for the below features on Mac OS 11.0 + // to detect their supported state dynamically. Assume the CPU features that + // Apple Silicon M1 supports to be available as a minimal set of features + // to all Go programs running on darwin/arm64. + ARM64.HasEVTSTRM = true + ARM64.HasAES = true + ARM64.HasPMULL = true + ARM64.HasSHA1 = true + ARM64.HasSHA2 = true + ARM64.HasCPUID = true + ARM64.HasASIMDRDM = true + ARM64.HasJSCVT = true + ARM64.HasFCMA = true + ARM64.HasLRCPC = true + ARM64.HasDCPOP = true + ARM64.HasSM3 = true + ARM64.HasSM4 = true + ARM64.HasASIMDDP = true +} + +const ( + commpage64_base_address = 0x0000000fffffc000 + commpage64_cpu_capabilities64 = (commpage64_base_address + 0x010) +) + +func darwinCheckFeatureEnabled(feature_vec uint64) bool { + if (*(*int64)(unsafe.Pointer(uintptr(commpage64_cpu_capabilities64))))&int64(feature_vec) > 0 { + return true + } + + return false +} diff --git a/cpu/cpu_linux_arm64.go b/cpu/cpu_linux_arm64.go index 79a38a0b9b..7f55a6ed89 100644 --- a/cpu/cpu_linux_arm64.go +++ b/cpu/cpu_linux_arm64.go @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build arm64 +// +build linux +// +build !android + package cpu // HWCAP/HWCAP2 bits. These are exposed by Linux. @@ -32,7 +36,7 @@ const ( hwcap_ASIMDFHM = 1 << 23 ) -func doinit() { +func osInit() { if err := readHWCAP(); err != nil { // failed to read /proc/self/auxv, try reading registers directly readARM64Registers() diff --git a/cpu/cpu_other_arm64.go b/cpu/cpu_other_arm64.go index f8c484f589..e5a23d1ac3 100644 --- a/cpu/cpu_other_arm64.go +++ b/cpu/cpu_other_arm64.go @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !linux && !netbsd && arm64 -// +build !linux,!netbsd,arm64 +//go:build !linux && !netbsd && !darwin && arm64 +// +build !linux,!netbsd,!darwin,arm64 package cpu -func doinit() {} +func osInit() { + setMinimalFeatures() +} diff --git a/cpu/cpu_test.go b/cpu/cpu_test.go index 5f7f843fe4..ba25551509 100644 --- a/cpu/cpu_test.go +++ b/cpu/cpu_test.go @@ -42,7 +42,7 @@ func TestAVX512HasAVX2AndAVX(t *testing.T) { } func TestARM64minimalFeatures(t *testing.T) { - if runtime.GOARCH != "arm64" || (runtime.GOOS == "darwin" || runtime.GOOS == "ios") { + if runtime.GOARCH != "arm64" || runtime.GOOS == "ios" { return } if !cpu.ARM64.HasASIMD {