| // Copyright 2020 Google LLC | |
| // Copyright 2024-2025 Arm Limited and/or its affiliates | |
| // <open-source-office@arm.com> | |
| // SPDX-License-Identifier: Apache-2.0 | |
| // SPDX-License-Identifier: BSD-3-Clause | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| // Sets macros based on HWY_TARGET. | |
| // This include guard is toggled by foreach_target, so avoid the usual _H_ | |
| // suffix to prevent copybara from renaming it. | |
| #if defined(HWY_SET_MACROS_PER_TARGET) == defined(HWY_TARGET_TOGGLE) | |
| #ifdef HWY_SET_MACROS_PER_TARGET | |
| #undef HWY_SET_MACROS_PER_TARGET | |
| #else | |
| #define HWY_SET_MACROS_PER_TARGET | |
| #endif | |
| #endif // HWY_SET_MACROS_PER_TARGET | |
| #include "hwy/detect_compiler_arch.h" // IWYU: export | |
| #include "hwy/detect_targets.h" // IWYU: export | |
| #undef HWY_NAMESPACE | |
| #undef HWY_ALIGN | |
| #undef HWY_MAX_BYTES | |
| #undef HWY_MIN_BYTES | |
| #undef HWY_HAVE_SCALABLE | |
| #undef HWY_HAVE_TUPLE | |
| #undef HWY_HAVE_INTEGER64 | |
| #undef HWY_HAVE_FLOAT16 | |
| #undef HWY_HAVE_FLOAT64 | |
| #undef HWY_MEM_OPS_MIGHT_FAULT | |
| #undef HWY_NATIVE_FMA | |
| #undef HWY_NATIVE_DOT_BF16 | |
| #undef HWY_NATIVE_MASK | |
| #undef HWY_NATIVE_INTERLEAVE_WHOLE | |
| #ifndef HWY_CAP_GE256 | |
| #define HWY_CAP_GE256 (HWY_MIN_BYTES >= 32) | |
| #endif | |
| #ifndef HWY_CAP_GE512 | |
| #define HWY_CAP_GE512 (HWY_MIN_BYTES >= 64) | |
| #endif | |
| // Almost all targets (except RVV and SCALAR) use this definition. | |
| #undef HWY_LANES | |
| #define HWY_LANES(T) (HWY_MAX_BYTES / sizeof(T)) | |
| #undef HWY_TARGET_IS_SVE | |
| #if HWY_TARGET & HWY_ALL_SVE | |
| #define HWY_TARGET_IS_SVE 1 | |
| #else | |
| #define HWY_TARGET_IS_SVE 0 | |
| #endif | |
| #undef HWY_TARGET_IS_NEON | |
| #if HWY_TARGET & HWY_ALL_NEON | |
| #define HWY_TARGET_IS_NEON 1 | |
| #else | |
| #define HWY_TARGET_IS_NEON 0 | |
| #endif | |
| #undef HWY_TARGET_IS_PPC | |
| #if HWY_TARGET & HWY_ALL_PPC | |
| #define HWY_TARGET_IS_PPC 1 | |
| #else | |
| #define HWY_TARGET_IS_PPC 0 | |
| #endif | |
| #undef HWY_TARGET_IS_AVX10_2 | |
| #if HWY_TARGET == HWY_AVX10_2 | |
| #define HWY_TARGET_IS_AVX10_2 1 | |
| #else | |
| #define HWY_TARGET_IS_AVX10_2 0 | |
| #endif | |
| // Supported on all targets except RVV (requires GCC 14 or upcoming Clang) | |
| #if HWY_TARGET == HWY_RVV && \ | |
| ((HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400) || \ | |
| (HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 1700)) | |
| #define HWY_HAVE_TUPLE 0 | |
| #else | |
| #define HWY_HAVE_TUPLE 1 | |
| #endif | |
| // For internal use (clamping/validating N for Simd<>) | |
| #undef HWY_MAX_N | |
| #if HWY_TARGET == HWY_SCALAR | |
| #define HWY_MAX_N 1 | |
| #else | |
| #define HWY_MAX_N 65536 | |
| #endif | |
| // For internal use (clamping kPow2 for Simd<>) | |
| #undef HWY_MAX_POW2 | |
| // For HWY_TARGET == HWY_RVV, LMUL <= 8. Even on other targets, we want to | |
| // support say Rebind<uint64_t, Simd<uint8_t, 1, 0>> d; whose kPow2 is also 3. | |
| // However, those other targets do not actually support multiple vectors, and | |
| // thus Lanes(d) must not exceed Lanes(ScalableTag<T>()). | |
| #define HWY_MAX_POW2 3 | |
| // User-visible. Loose lower bound that guarantees HWY_MAX_BYTES >> | |
| // (-HWY_MIN_POW2) <= 1. Useful for terminating compile-time recursions. | |
| #undef HWY_MIN_POW2 | |
| #if HWY_TARGET == HWY_RVV | |
| #define HWY_MIN_POW2 -16 | |
| #else | |
| // Tighter bound for other targets, whose vectors are smaller, to potentially | |
| // save compile time. | |
| #define HWY_MIN_POW2 -8 | |
| #endif // HWY_TARGET == HWY_RVV | |
| #undef HWY_TARGET_STR | |
| #if defined(HWY_DISABLE_PCLMUL_AES) | |
| #define HWY_TARGET_STR_PCLMUL_AES "" | |
| #else | |
| #define HWY_TARGET_STR_PCLMUL_AES ",pclmul,aes" | |
| #endif | |
| #if defined(HWY_DISABLE_BMI2_FMA) | |
| #define HWY_TARGET_STR_BMI2_FMA "" | |
| #else | |
| #define HWY_TARGET_STR_BMI2_FMA ",bmi,bmi2,fma" | |
| #endif | |
| #if defined(HWY_DISABLE_F16C) | |
| #define HWY_TARGET_STR_F16C "" | |
| #else | |
| #define HWY_TARGET_STR_F16C ",f16c" | |
| #endif | |
| #define HWY_TARGET_STR_SSE2 "sse2" | |
| #define HWY_TARGET_STR_SSSE3 "sse2,ssse3" | |
| #define HWY_TARGET_STR_SSE4 \ | |
| HWY_TARGET_STR_SSSE3 ",sse4.1,sse4.2" HWY_TARGET_STR_PCLMUL_AES | |
| // Include previous targets, which are the half-vectors of the next target. | |
| #define HWY_TARGET_STR_AVX2 \ | |
| HWY_TARGET_STR_SSE4 ",avx,avx2" HWY_TARGET_STR_BMI2_FMA HWY_TARGET_STR_F16C | |
| #ifndef HWY_HAVE_EVEX512 // allow override | |
| // evex512 has been removed from clang 22, see | |
| // https://github.com/llvm/llvm-project/pull/157034 | |
| #if (1400 <= HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1600) || \ | |
| (1800 <= HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 2200) | |
| #define HWY_HAVE_EVEX512 1 | |
| #else | |
| #define HWY_HAVE_EVEX512 0 | |
| #endif | |
| #endif | |
| #if (HWY_HAVE_EVEX512 == 1) | |
| #define HWY_TARGET_STR_AVX3_VL512 ",evex512" | |
| #else | |
| #define HWY_TARGET_STR_AVX3_VL512 | |
| #endif | |
| #define HWY_TARGET_STR_AVX3 \ | |
| HWY_TARGET_STR_AVX2 \ | |
| ",avx512f,avx512cd,avx512vl,avx512dq,avx512bw" HWY_TARGET_STR_AVX3_VL512 | |
| #define HWY_TARGET_STR_AVX3_DL \ | |
| HWY_TARGET_STR_AVX3 \ | |
| ",vpclmulqdq,avx512vbmi,avx512vbmi2,vaes,avx512vnni,avx512bitalg," \ | |
| "avx512vpopcntdq,gfni" | |
| // Opt-out for compilers that do not properly support avx512bf16. | |
| #ifndef HWY_AVX3_ENABLE_AVX512BF16 // allow override | |
| // Default is to disable if the DISABLE macro is defined, or if old compiler. | |
| // clang-cl 21.1.4 reportedly works; feel free to define this to 1 there. | |
| #if defined(HWY_AVX3_DISABLE_AVX512BF16) || \ | |
| (HWY_COMPILER_CLANGCL || \ | |
| (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1000) || \ | |
| (HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 900)) | |
| #define HWY_AVX3_ENABLE_AVX512BF16 0 | |
| #else | |
| #define HWY_AVX3_ENABLE_AVX512BF16 1 | |
| #endif | |
| #endif // HWY_AVX3_ENABLE_AVX512BF16 | |
| #if HWY_AVX3_ENABLE_AVX512BF16 | |
| #define HWY_TARGET_STR_AVX3_ZEN4 HWY_TARGET_STR_AVX3_DL ",avx512bf16" | |
| #else | |
| #define HWY_TARGET_STR_AVX3_ZEN4 HWY_TARGET_STR_AVX3_DL | |
| #endif | |
| #if HWY_COMPILER_GCC_ACTUAL >= 1200 || HWY_COMPILER_CLANG >= 1400 | |
| #define HWY_TARGET_STR_AVX3_SPR HWY_TARGET_STR_AVX3_ZEN4 ",avx512fp16" | |
| #else | |
| #define HWY_TARGET_STR_AVX3_SPR HWY_TARGET_STR_AVX3_ZEN4 | |
| #endif | |
| // Support for avx10.2-512 was removed between clang 22 and 23 without a | |
| // feature test macro. | |
| #if HWY_COMPILER_CLANG >= 2200 && HWY_HAVE_EVEX512 | |
| #define HWY_TARGET_STR_AVX10_2 HWY_TARGET_STR_AVX3_SPR ",avx10.2-512" | |
| // Recent compilers drop the -512 suffix because 512 bits are always available. | |
| #elif HWY_COMPILER_GCC_ACTUAL >= 1500 || HWY_COMPILER_CLANG >= 2200 | |
| #define HWY_TARGET_STR_AVX10_2 HWY_TARGET_STR_AVX3_SPR ",avx10.2" | |
| #else | |
| #define HWY_TARGET_STR_AVX10_2 HWY_TARGET_STR_AVX3_SPR | |
| #endif | |
| #if defined(HWY_DISABLE_PPC8_CRYPTO) | |
| #define HWY_TARGET_STR_PPC8_CRYPTO "" | |
| #else | |
| #define HWY_TARGET_STR_PPC8_CRYPTO ",crypto" | |
| #endif | |
| #define HWY_TARGET_STR_PPC8 \ | |
| "altivec,vsx,power8-vector" HWY_TARGET_STR_PPC8_CRYPTO | |
| #define HWY_TARGET_STR_PPC9 HWY_TARGET_STR_PPC8 ",power9-vector" | |
| #if HWY_COMPILER_CLANG | |
| #define HWY_TARGET_STR_PPC10 HWY_TARGET_STR_PPC9 ",power10-vector" | |
| #else | |
| // See #1707 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102059#c35. | |
| // When the baseline is PPC 8 or 9, inlining functions such as PreventElision | |
| // into PPC10 code fails because PPC10 defaults to no-htm and is thus worse than | |
| // the baseline, which has htm. We cannot have pragma target on functions | |
| // outside HWY_NAMESPACE such as those in base.h. It would be possible for users | |
| // to set -mno-htm globally, but we can also work around this at the library | |
| // level by claiming that PPC10 still has HTM, thus avoiding the mismatch. This | |
| // seems to be safe because HTM uses builtins rather than modifying codegen, see | |
| // https://gcc.gnu.org/legacy-ml/gcc-patches/2013-07/msg00167.html. | |
| #define HWY_TARGET_STR_PPC10 HWY_TARGET_STR_PPC9 ",cpu=power10,htm" | |
| #endif | |
| #define HWY_TARGET_STR_Z14 "arch=z14" | |
| #define HWY_TARGET_STR_Z15 "arch=z15" | |
| // Before include guard so we redefine HWY_TARGET_STR on each include, | |
| // governed by the current HWY_TARGET. | |
| //----------------------------------------------------------------------------- | |
| // SSE2 | |
| #if HWY_TARGET == HWY_SSE2 | |
| #define HWY_NAMESPACE N_SSE2 | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 // a few actually are | |
| #define HWY_TARGET_STR HWY_TARGET_STR_SSE2 | |
| //----------------------------------------------------------------------------- | |
| // SSSE3 | |
| #elif HWY_TARGET == HWY_SSSE3 | |
| #define HWY_NAMESPACE N_SSSE3 | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 // a few actually are | |
| #define HWY_TARGET_STR HWY_TARGET_STR_SSSE3 | |
| //----------------------------------------------------------------------------- | |
| // SSE4 | |
| #elif HWY_TARGET == HWY_SSE4 | |
| #define HWY_NAMESPACE N_SSE4 | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 // a few actually are | |
| #define HWY_TARGET_STR HWY_TARGET_STR_SSE4 | |
| //----------------------------------------------------------------------------- | |
| // AVX2 | |
| #elif HWY_TARGET == HWY_AVX2 | |
| #define HWY_NAMESPACE N_AVX2 | |
| #define HWY_ALIGN alignas(32) | |
| #define HWY_MAX_BYTES 32 | |
| #define HWY_MIN_BYTES 32 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #ifdef HWY_DISABLE_BMI2_FMA | |
| #define HWY_NATIVE_FMA 0 | |
| #else | |
| #define HWY_NATIVE_FMA 1 | |
| #endif | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 // a few actually are | |
| #define HWY_TARGET_STR HWY_TARGET_STR_AVX2 | |
| //----------------------------------------------------------------------------- | |
| // AVX3[_DL/ZEN4/SPR]/AVX10 | |
| #elif HWY_TARGET <= HWY_AVX3 | |
| #define HWY_ALIGN alignas(64) | |
| #define HWY_MAX_BYTES 64 | |
| #define HWY_MIN_BYTES 64 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #if HWY_TARGET <= HWY_AVX3_SPR && \ | |
| (HWY_COMPILER_GCC_ACTUAL || HWY_COMPILER_CLANG >= 2200) && \ | |
| HWY_HAVE_SCALAR_F16_TYPE | |
| #define HWY_HAVE_FLOAT16 1 | |
| #else | |
| #define HWY_HAVE_FLOAT16 0 | |
| #endif | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 0 | |
| #define HWY_NATIVE_FMA 1 | |
| #if (HWY_TARGET <= HWY_AVX3_ZEN4) && HWY_AVX3_ENABLE_AVX512BF16 | |
| #define HWY_NATIVE_DOT_BF16 1 | |
| #else | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #endif | |
| #define HWY_NATIVE_MASK 1 | |
| #if HWY_TARGET == HWY_AVX3 | |
| #define HWY_NAMESPACE N_AVX3 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_AVX3 | |
| #elif HWY_TARGET == HWY_AVX3_DL | |
| #define HWY_NAMESPACE N_AVX3_DL | |
| #define HWY_TARGET_STR HWY_TARGET_STR_AVX3_DL | |
| #elif HWY_TARGET == HWY_AVX3_ZEN4 | |
| #define HWY_NAMESPACE N_AVX3_ZEN4 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_AVX3_ZEN4 | |
| #elif HWY_TARGET == HWY_AVX3_SPR | |
| #define HWY_NAMESPACE N_AVX3_SPR | |
| #define HWY_TARGET_STR HWY_TARGET_STR_AVX3_SPR | |
| #elif HWY_TARGET == HWY_AVX10_2 | |
| #define HWY_NAMESPACE N_AVX10_2 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_AVX10_2 | |
| #else | |
| #error "Logic error" | |
| #endif // HWY_TARGET | |
| //----------------------------------------------------------------------------- | |
| // PPC8, PPC9, PPC10 | |
| #elif HWY_TARGET_IS_PPC | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 1 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #if HWY_TARGET == HWY_PPC8 | |
| #define HWY_NAMESPACE N_PPC8 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_PPC8 | |
| #elif HWY_TARGET == HWY_PPC9 | |
| #define HWY_NAMESPACE N_PPC9 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_PPC9 | |
| #elif HWY_TARGET == HWY_PPC10 | |
| #define HWY_NAMESPACE N_PPC10 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_PPC10 | |
| #else | |
| #error "Logic error" | |
| #endif // HWY_TARGET | |
| //----------------------------------------------------------------------------- | |
| // Z14, Z15 | |
| #elif HWY_TARGET == HWY_Z14 || HWY_TARGET == HWY_Z15 | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 1 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #if HWY_TARGET == HWY_Z14 | |
| #define HWY_NAMESPACE N_Z14 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_Z14 | |
| #elif HWY_TARGET == HWY_Z15 | |
| #define HWY_NAMESPACE N_Z15 | |
| #define HWY_TARGET_STR HWY_TARGET_STR_Z15 | |
| #else | |
| #error "Logic error" | |
| #endif // HWY_TARGET == HWY_Z15 | |
| //----------------------------------------------------------------------------- | |
| // NEON | |
| #elif HWY_TARGET_IS_NEON | |
| // Clang 17 crashes with bf16, see github.com/llvm/llvm-project/issues/64179. | |
| #undef HWY_NEON_HAVE_BFLOAT16 | |
| #if HWY_HAVE_SCALAR_BF16_TYPE && \ | |
| ((HWY_TARGET == HWY_NEON_BF16 && \ | |
| (!HWY_COMPILER_CLANG || HWY_COMPILER_CLANG >= 1800)) || \ | |
| defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)) | |
| #define HWY_NEON_HAVE_BFLOAT16 1 | |
| #else | |
| #define HWY_NEON_HAVE_BFLOAT16 0 | |
| #endif | |
| // HWY_NEON_HAVE_F32_TO_BF16C is defined if NEON vcvt_bf16_f32 and | |
| // vbfdot_f32 are available, even if the __bf16 type is disabled due to | |
| // GCC/Clang bugs. | |
| #undef HWY_NEON_HAVE_F32_TO_BF16C | |
| #if HWY_NEON_HAVE_BFLOAT16 || HWY_TARGET == HWY_NEON_BF16 || \ | |
| (defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && \ | |
| (HWY_COMPILER_GCC_ACTUAL >= 1000 || HWY_COMPILER_CLANG >= 1100)) | |
| #define HWY_NEON_HAVE_F32_TO_BF16C 1 | |
| #else | |
| #define HWY_NEON_HAVE_F32_TO_BF16C 0 | |
| #endif | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || HWY_TARGET == HWY_NEON_BF16 | |
| #define HWY_HAVE_FLOAT16 1 | |
| #else | |
| #define HWY_HAVE_FLOAT16 0 | |
| #endif | |
| #if HWY_ARCH_ARM_A64 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #else | |
| #define HWY_HAVE_FLOAT64 0 | |
| #endif | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #if defined(__ARM_FEATURE_FMA) || defined(__ARM_VFPV4__) || HWY_ARCH_ARM_A64 | |
| #define HWY_NATIVE_FMA 1 | |
| #else | |
| #define HWY_NATIVE_FMA 0 | |
| #endif | |
| #if HWY_NEON_HAVE_F32_TO_BF16C | |
| #define HWY_NATIVE_DOT_BF16 1 | |
| #else | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #endif | |
| #define HWY_NATIVE_MASK 0 | |
| #if HWY_TARGET == HWY_NEON_WITHOUT_AES | |
| #define HWY_NAMESPACE N_NEON_WITHOUT_AES | |
| #elif HWY_TARGET == HWY_NEON | |
| #define HWY_NAMESPACE N_NEON | |
| #elif HWY_TARGET == HWY_NEON_BF16 | |
| #define HWY_NAMESPACE N_NEON_BF16 | |
| #else | |
| #error "Logic error, missing case" | |
| #endif // HWY_TARGET | |
| // Can use pragmas instead of -march compiler flag | |
| #if HWY_HAVE_RUNTIME_DISPATCH | |
| #if HWY_ARCH_ARM_V7 | |
| // The __attribute__((target(+neon-vfpv4)) was introduced in gcc >= 8. | |
| #if HWY_COMPILER_GCC_ACTUAL >= 800 | |
| #define HWY_TARGET_STR "+neon-vfpv4" | |
| #else // GCC < 7 | |
| // Do not define HWY_TARGET_STR (no pragma). | |
| #endif // HWY_COMPILER_GCC_ACTUAL | |
| #else // !HWY_ARCH_ARM_V7 | |
| #if (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1300) || \ | |
| (HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 1300) | |
| // GCC 12 or earlier and Clang 12 or earlier require +crypto be added to the | |
| // target string to enable AArch64 AES intrinsics | |
| #define HWY_TARGET_STR_NEON "+crypto" | |
| #else | |
| #define HWY_TARGET_STR_NEON "+aes" | |
| #endif | |
| // Clang >= 16 requires +fullfp16 instead of fp16, but Apple Clang 15 = 1600 | |
| // fails to parse unless the string starts with armv8, whereas 1700 refuses it. | |
| #if HWY_COMPILER_CLANG >= 1700 | |
| #define HWY_TARGET_STR_FP16 "+fullfp16" | |
| #elif HWY_COMPILER_CLANG >= 1600 && defined(__apple_build_version__) | |
| #define HWY_TARGET_STR_FP16 "armv8.4-a+fullfp16" | |
| #else | |
| #define HWY_TARGET_STR_FP16 "+fp16" | |
| #endif | |
| #if HWY_TARGET == HWY_NEON_WITHOUT_AES | |
| #if HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400 | |
| // Prevents inadvertent use of SVE by GCC 13.4 and earlier, see #2689. | |
| #define HWY_TARGET_STR "+nosve" | |
| #else | |
| // Do not define HWY_TARGET_STR (no pragma). | |
| #endif // HWY_COMPILER_GCC_ACTUAL | |
| #elif HWY_TARGET == HWY_NEON | |
| #define HWY_TARGET_STR HWY_TARGET_STR_NEON | |
| #elif HWY_TARGET == HWY_NEON_BF16 | |
| #define HWY_TARGET_STR \ | |
| HWY_TARGET_STR_FP16 "+bf16+dotprod" HWY_TARGET_STR_NEON | |
| #else | |
| #error "Logic error, missing case" | |
| #endif // HWY_TARGET | |
| #endif // !HWY_ARCH_ARM_V7 | |
| #else // !HWY_HAVE_RUNTIME_DISPATCH | |
| // HWY_TARGET_STR remains undefined | |
| #endif | |
| //----------------------------------------------------------------------------- | |
| // SVE[2] | |
| #elif HWY_TARGET_IS_SVE | |
| // SVE only requires lane alignment, not natural alignment of the entire vector. | |
| #define HWY_ALIGN alignas(8) | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 1 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 0 | |
| #define HWY_NATIVE_FMA 1 | |
| #if HWY_SVE_HAVE_BF16_FEATURE | |
| #define HWY_NATIVE_DOT_BF16 1 | |
| #else | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #endif | |
| #define HWY_NATIVE_MASK 1 | |
| #if HWY_TARGET == HWY_SVE2 | |
| #define HWY_NAMESPACE N_SVE2 | |
| #define HWY_MAX_BYTES 256 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 1 | |
| #elif HWY_TARGET == HWY_SVE_256 | |
| #define HWY_NAMESPACE N_SVE_256 | |
| #define HWY_MAX_BYTES 32 | |
| #define HWY_MIN_BYTES 32 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #elif HWY_TARGET == HWY_SVE2_128 | |
| #define HWY_NAMESPACE N_SVE2_128 | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #else | |
| #define HWY_NAMESPACE N_SVE | |
| #define HWY_MAX_BYTES 256 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 1 | |
| #endif | |
| // Can use pragmas instead of -march compiler flag | |
| #if HWY_HAVE_RUNTIME_DISPATCH | |
| #if HWY_TARGET == HWY_SVE2 || HWY_TARGET == HWY_SVE2_128 | |
| // Static dispatch with -march=armv8-a+sve2+aes, or no baseline, hence dynamic | |
| // dispatch, which checks for AES support at runtime. | |
| #if defined(__ARM_FEATURE_SVE2_AES) || (HWY_BASELINE_SVE2 == 0) | |
| #define HWY_TARGET_STR "+sve2+sve2-aes,+sve" | |
| #else // SVE2 without AES | |
| #define HWY_TARGET_STR "+sve2,+sve" | |
| #endif | |
| #else // not SVE2 target | |
| #define HWY_TARGET_STR "+sve" | |
| #endif | |
| #else // !HWY_HAVE_RUNTIME_DISPATCH | |
| // HWY_TARGET_STR remains undefined | |
| #endif | |
| //----------------------------------------------------------------------------- | |
| // WASM | |
| #elif HWY_TARGET == HWY_WASM | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #define HWY_NAMESPACE N_WASM | |
| #define HWY_TARGET_STR "simd128" | |
| //----------------------------------------------------------------------------- | |
| // WASM_EMU256 | |
| #elif HWY_TARGET == HWY_WASM_EMU256 | |
| #define HWY_ALIGN alignas(32) | |
| #define HWY_MAX_BYTES 32 | |
| #define HWY_MIN_BYTES 32 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #define HWY_NAMESPACE N_WASM_EMU256 | |
| #define HWY_TARGET_STR "simd128" | |
| //----------------------------------------------------------------------------- | |
| // RVV | |
| #elif HWY_TARGET == HWY_RVV | |
| // RVV only requires lane alignment, not natural alignment of the entire vector, | |
| // and the compiler already aligns builtin types, so nothing to do here. | |
| #define HWY_ALIGN | |
| // The spec requires VLEN <= 2^16 bits, so the limit is 2^16 bytes (LMUL=8). | |
| #define HWY_MAX_BYTES 65536 | |
| #define HWY_MIN_BYTES 16 | |
| // = HWY_MAX_BYTES divided by max LMUL=8 because MaxLanes includes the actual | |
| // LMUL. This is the tightest possible upper bound. | |
| #undef HWY_LANES | |
| #define HWY_LANES(T) (8192 / sizeof(T)) | |
| #define HWY_HAVE_SCALABLE 1 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 0 | |
| #define HWY_NATIVE_FMA 1 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 1 | |
| #if HWY_RVV_HAVE_F16_VEC | |
| #define HWY_HAVE_FLOAT16 1 | |
| #else | |
| #define HWY_HAVE_FLOAT16 0 | |
| #endif | |
| #define HWY_NAMESPACE N_RVV | |
| #if HWY_COMPILER_CLANG >= 1900 | |
| // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#181-zvl-minimum-vector-length-standard-extensions | |
| #define HWY_TARGET_STR "arch=+v" | |
| #else | |
| // HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. | |
| #endif | |
| //----------------------------------------------------------------------------- | |
| // LSX/LASX | |
| #elif HWY_TARGET == HWY_LSX || HWY_TARGET == HWY_LASX | |
| #if HWY_TARGET == HWY_LSX | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #ifndef __loongarch_sx | |
| #define HWY_TARGET_STR "lsx" | |
| #endif | |
| #else | |
| #define HWY_ALIGN alignas(32) | |
| #define HWY_MAX_BYTES 32 | |
| #define HWY_MIN_BYTES 32 | |
| #ifndef __loongarch_asx | |
| #define HWY_TARGET_STR "lsx,lasx" | |
| #endif | |
| #endif | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 1 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #if HWY_TARGET == HWY_LSX | |
| #define HWY_NAMESPACE N_LSX | |
| #else | |
| #define HWY_NAMESPACE N_LASX | |
| #endif | |
| // HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. | |
| //----------------------------------------------------------------------------- | |
| // EMU128 | |
| #elif HWY_TARGET == HWY_EMU128 | |
| #define HWY_ALIGN alignas(16) | |
| #define HWY_MAX_BYTES 16 | |
| #define HWY_MIN_BYTES 16 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #define HWY_NAMESPACE N_EMU128 | |
| // HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. | |
| //----------------------------------------------------------------------------- | |
| // SCALAR | |
| #elif HWY_TARGET == HWY_SCALAR | |
| #define HWY_ALIGN | |
| #define HWY_MAX_BYTES 8 | |
| #define HWY_MIN_BYTES 8 | |
| #undef HWY_LANES | |
| #define HWY_LANES(T) 1 | |
| #define HWY_HAVE_SCALABLE 0 | |
| #define HWY_HAVE_INTEGER64 1 | |
| #define HWY_HAVE_FLOAT16 0 | |
| #define HWY_HAVE_FLOAT64 1 | |
| #define HWY_MEM_OPS_MIGHT_FAULT 0 | |
| #define HWY_NATIVE_FMA 0 | |
| #define HWY_NATIVE_DOT_BF16 0 | |
| #define HWY_NATIVE_MASK 0 | |
| #define HWY_NAMESPACE N_SCALAR | |
| // HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. | |
| #else | |
| #pragma message("HWY_TARGET does not match any known target") | |
| #endif // HWY_TARGET | |
| //----------------------------------------------------------------------------- | |
| // Sanity check: if we have f16 vector support, then base.h should also be | |
| // using a built-in type for f16 scalars. | |
| #if HWY_HAVE_FLOAT16 && !HWY_HAVE_SCALAR_F16_TYPE | |
| #error "Logic error: f16 vectors but no scalars" | |
| #endif | |
| // Override this to 1 in asan/msan builds, which will still fault. | |
| #if HWY_IS_ASAN || HWY_IS_MSAN | |
| #undef HWY_MEM_OPS_MIGHT_FAULT | |
| #define HWY_MEM_OPS_MIGHT_FAULT 1 | |
| #endif | |
| // Clang <9 requires this be invoked at file scope, before any namespace. | |
| #undef HWY_BEFORE_NAMESPACE | |
| #if defined(HWY_TARGET_STR) && !defined(HWY_DISABLE_ATTR) | |
| #define HWY_BEFORE_NAMESPACE() \ | |
| HWY_PUSH_ATTRIBUTES(HWY_TARGET_STR) \ | |
| static_assert(true, "For requiring trailing semicolon") | |
| #else | |
| // avoids compiler warning if no HWY_TARGET_STR | |
| #define HWY_BEFORE_NAMESPACE() \ | |
| static_assert(true, "For requiring trailing semicolon") | |
| #endif | |
| // Clang <9 requires any namespaces be closed before this macro. | |
| #undef HWY_AFTER_NAMESPACE | |
| #if defined(HWY_TARGET_STR) && !defined(HWY_DISABLE_ATTR) | |
| #define HWY_AFTER_NAMESPACE() \ | |
| HWY_POP_ATTRIBUTES \ | |
| static_assert(true, "For requiring trailing semicolon") | |
| #else | |
| // avoids compiler warning if no HWY_TARGET_STR | |
| #define HWY_AFTER_NAMESPACE() \ | |
| static_assert(true, "For requiring trailing semicolon") | |
| #endif | |
| #undef HWY_ATTR | |
| #if defined(HWY_TARGET_STR) && HWY_HAS_ATTRIBUTE(target) && \ | |
| !defined(HWY_DISABLE_ATTR) | |
| #define HWY_ATTR __attribute__((target(HWY_TARGET_STR))) | |
| #else | |
| #define HWY_ATTR | |
| #endif | |
| #if (HWY_MAX_BYTES <= 16) || HWY_TARGET_IS_SVE || (HWY_TARGET == HWY_RVV) || \ | |
| (HWY_TARGET == HWY_WASM_EMU256) | |
| #define HWY_NATIVE_INTERLEAVE_WHOLE 1 | |
| #else | |
| #define HWY_NATIVE_INTERLEAVE_WHOLE 0 | |
| #endif |