| // Copyright 2023 Google LLC | |
| // SPDX-License-Identifier: Apache-2.0 | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #undef HWY_TARGET_INCLUDE | |
| #define HWY_TARGET_INCLUDE "tests/foreach_vec_test.cc" | |
| #include "hwy/foreach_target.h" // IWYU pragma: keep | |
| #include "hwy/highway.h" | |
| #include "hwy/tests/test_util-inl.h" | |
| HWY_BEFORE_NAMESPACE(); | |
| namespace hwy { | |
| namespace HWY_NAMESPACE { | |
| namespace { | |
| struct ForeachVectorTestPerLaneSizeState { | |
| size_t num_of_lanes_mask; | |
| #if HWY_HAVE_SCALABLE | |
| int pow2_mask; | |
| #endif | |
| }; | |
| struct ForeachVectorTestState { | |
| ForeachVectorTestPerLaneSizeState per_lane_size_states[16]; | |
| int lane_sizes_mask; | |
| }; | |
| template <class D> | |
| static HWY_INLINE void UpdateForeachVectorTestState( | |
| ForeachVectorTestState &state, D d) { | |
| using T = TFromD<D>; | |
| static_assert(sizeof(T) >= 1 && sizeof(T) <= 8, | |
| "sizeof(T) must be between 1 and 8"); | |
| state.lane_sizes_mask |= (1 << sizeof(T)); | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &state.per_lane_size_states[sizeof(T)]; | |
| const size_t lanes = Lanes(d); | |
| HWY_ASSERT(lanes > 0 && (lanes & (lanes - 1)) == 0); | |
| per_lane_size_state->num_of_lanes_mask |= lanes; | |
| #if HWY_HAVE_SCALABLE | |
| constexpr int kPow2 = D().Pow2(); | |
| #if HWY_TARGET == HWY_RVV | |
| static_assert(kPow2 >= detail::MinPow2<T>(), | |
| "kPow2 >= detail::MinPow2<T>() must be true"); | |
| #endif | |
| static_assert(kPow2 <= detail::MaxPow2(), | |
| "kPow2 <= detail::MaxPow2() must be true"); | |
| if (HWY_TARGET == HWY_RVV || kPow2 >= -3) { | |
| per_lane_size_state->pow2_mask |= (1 << (kPow2 + 3)); | |
| } | |
| #endif | |
| } | |
| static constexpr int kMaxSupportedLaneSize = HWY_HAVE_INTEGER64 ? 8 : 4; | |
| static constexpr int kSupportedLaneSizesMask = | |
| (1 << 1) | (1 << 2) | (1 << 4) | (HWY_HAVE_INTEGER64 ? (1 << 8) : 0); | |
| #if HWY_HAVE_SCALABLE | |
| static constexpr int kSupportedU8Pow2Mask = | |
| (HWY_TARGET == HWY_RVV) ? 0x7F : 0x0F; | |
| #endif | |
| static HWY_INLINE size_t LanesPerVectWithLaneSize(size_t lanes_per_u8_vect, | |
| int lane_size) { | |
| #if HWY_TARGET == HWY_SCALAR | |
| (void)lanes_per_u8_vect; | |
| (void)lane_size; | |
| return 1; | |
| #else | |
| return lanes_per_u8_vect / static_cast<size_t>(lane_size); | |
| #endif | |
| } | |
| #define HWY_DECLARE_FOREACH_VECTOR_TEST(TestClass) \ | |
| static ForeachVectorTestState TestClass##State; \ | |
| \ | |
| struct TestClass { \ | |
| template <class T, class D> \ | |
| HWY_INLINE void operator()(T, D d) { \ | |
| UpdateForeachVectorTestState(TestClass##State, d); \ | |
| } \ | |
| }; | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForMaxPow2) | |
| HWY_NOINLINE void TestAllForMaxPow2() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForMaxPow2State); | |
| ForUnsignedTypes(ForMaxPow2<TestForMaxPow2>()); | |
| HWY_ASSERT(TestForMaxPow2State.lane_sizes_mask == kSupportedLaneSizesMask); | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForMaxPow2State.per_lane_size_states[lane_size]; | |
| const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == | |
| ((lanes << (HWY_TARGET == HWY_RVV ? 2 : 1)) - 1)); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| (kSupportedU8Pow2Mask & 0x1F) & (-((lane_size + 1) / 2)); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForExtendableVectors) | |
| #if HWY_TARGET == HWY_RVV | |
| template <int kPow2, class Test, class T, | |
| hwy::EnableIf<(-kPow2 < detail::MinPow2<T>())> * = nullptr> | |
| static HWY_INLINE void ExecuteTestForExtendableVectors(const Test & /*test*/, | |
| T /*unused*/) {} | |
| template <int kPow2, class Test, class T, | |
| hwy::EnableIf<(-kPow2 >= detail::MinPow2<T>())> * = nullptr> | |
| static HWY_INLINE void ExecuteTestForExtendableVectors(const Test &test, | |
| T /*unused*/) { | |
| test(T()); | |
| } | |
| #endif | |
| template <int kPow2> | |
| static HWY_NOINLINE void DoTestAllForExtendableVectors() { | |
| static_assert(kPow2 >= 0 && kPow2 <= 3, "kPow2 must be between 0 and 3"); | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForExtendableVectorsState); | |
| const ForExtendableVectors<TestForExtendableVectors, kPow2> test; | |
| #if HWY_TARGET == HWY_RVV | |
| test(uint8_t()); | |
| ExecuteTestForExtendableVectors<kPow2>(test, uint16_t()); | |
| ExecuteTestForExtendableVectors<kPow2>(test, uint32_t()); | |
| ExecuteTestForExtendableVectors<kPow2>(test, uint64_t()); | |
| #else | |
| ForUnsignedTypes(test); | |
| #endif | |
| #if HWY_TARGET == HWY_SCALAR | |
| HWY_ASSERT(TestForExtendableVectorsState.lane_sizes_mask == 0); | |
| #else // HWY_TARGET != HWY_SCALAR | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t, -kPow2>()); | |
| #if HWY_TARGET == HWY_RVV | |
| const int expected_lane_sizes_mask = | |
| kSupportedLaneSizesMask & | |
| ((1 << 1) | (1 << 2) | ((kPow2 <= 2) ? (1 << 4) : 0) | | |
| ((kPow2 <= 1) ? (1 << 8) : 0)); | |
| #else | |
| const int expected_lane_sizes_mask = | |
| kSupportedLaneSizesMask & (((lanes_per_u8_vect >= 1) ? (1 << 1) : 0) | | |
| ((lanes_per_u8_vect >= 2) ? (1 << 2) : 0) | | |
| ((lanes_per_u8_vect >= 4) ? (1 << 4) : 0) | | |
| ((lanes_per_u8_vect >= 8) ? (1 << 8) : 0)); | |
| #endif | |
| HWY_ASSERT(TestForExtendableVectorsState.lane_sizes_mask == | |
| expected_lane_sizes_mask); | |
| #endif // HWY_TARGET == HWY_SCALAR | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForExtendableVectorsState.per_lane_size_states[lane_size]; | |
| #if HWY_TARGET == HWY_SCALAR | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #else | |
| if ((expected_lane_sizes_mask & (1 << lane_size)) != 0) { | |
| const size_t lanes = | |
| LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == | |
| ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1)); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| ((kSupportedU8Pow2Mask >> kPow2) & (-((lane_size + 1) / 2))) | | |
| ((HWY_TARGET == HWY_RVV) ? 0 : (1 << (3 - kPow2))); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } else { | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #if HWY_HAVE_SCALABLE | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == 0); | |
| #endif | |
| } | |
| #endif // HWY_TARGET == HWY_SCALAR | |
| } | |
| } | |
| HWY_NOINLINE void TestAllForExtendableVectors() { | |
| DoTestAllForExtendableVectors<1>(); | |
| DoTestAllForExtendableVectors<2>(); | |
| DoTestAllForExtendableVectors<3>(); | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForShrinkableVectors) | |
| HWY_NOINLINE void TestAllForShrinkableVectors() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForShrinkableVectorsState); | |
| ForUnsignedTypes(ForShrinkableVectors<TestForShrinkableVectors>()); | |
| #if HWY_TARGET == HWY_SCALAR | |
| HWY_ASSERT(TestForShrinkableVectorsState.lane_sizes_mask == 0); | |
| #else // HWY_TARGET != HWY_SCALAR | |
| HWY_ASSERT(TestForShrinkableVectorsState.lane_sizes_mask == | |
| kSupportedLaneSizesMask); | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| #endif // HWY_TARGET == HWY_SCALAR | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForShrinkableVectorsState.per_lane_size_states[lane_size]; | |
| #if HWY_TARGET == HWY_SCALAR | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #else // HWY_TARGET != HWY_SCALAR | |
| const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| kSupportedU8Pow2Mask & (-2 * ((lane_size + 1) / 2)); | |
| const size_t expected_lanes_mask = | |
| (lanes * static_cast<size_t>(expected_pow2_mask)) >> 3; | |
| HWY_ASSERT((per_lane_size_state->num_of_lanes_mask & expected_lanes_mask) == | |
| expected_lanes_mask); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #else // !HWY_HAVE_SCALABLE | |
| const size_t expected_lanes_mask = | |
| static_cast<size_t>(((lanes << 1) - 1) & (~size_t{1})); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == expected_lanes_mask); | |
| #endif // HWY_HAVE_SCALABLE | |
| #endif // HWY_TARGET == HWY_SCALAR | |
| } | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForGEVectors) | |
| template <size_t kMinBits, class Test, class T, | |
| HWY_IF_LANES_LE(kMinBits, sizeof(T) * 8 - 1)> | |
| static HWY_INLINE void ExecuteTestForGEVectors(const Test & /*test*/, | |
| T /*unused*/) {} | |
| template <size_t kMinBits, class Test, class T, | |
| HWY_IF_LANES_GT(kMinBits, sizeof(T) * 8 - 1)> | |
| static HWY_INLINE void ExecuteTestForGEVectors(const Test &test, T /*unused*/) { | |
| test(T()); | |
| } | |
| template <size_t kMinBits, class Test> | |
| static HWY_NOINLINE void DoTestAllForGEVectors(const Test &test) { | |
| static_assert(kMinBits >= 16, "kMinBits >= 16 must be true"); | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForGEVectorsState); | |
| test(uint8_t()); | |
| test(uint16_t()); | |
| ExecuteTestForGEVectors<kMinBits>(test, uint32_t()); | |
| #if HWY_HAVE_INTEGER64 | |
| ExecuteTestForGEVectors<kMinBits>(test, uint64_t()); | |
| #endif | |
| #if HWY_TARGET == HWY_SCALAR | |
| HWY_ASSERT(TestForGEVectorsState.lane_sizes_mask == 0); | |
| #else // HWY_TARGET != HWY_SCALAR | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| #if HWY_TARGET == HWY_RVV | |
| const size_t lanes_per_largest_u8_vect = lanes_per_u8_vect * 8; | |
| #else | |
| const size_t lanes_per_largest_u8_vect = lanes_per_u8_vect; | |
| #endif // HWY_TARGET == HWY_RVV | |
| constexpr int kGEVectSupportedLaneSizesMask = | |
| kSupportedLaneSizesMask & | |
| ((1 << 1) | (1 << 2) | ((kMinBits >= 32) ? (1 << 4) : 0) | | |
| ((kMinBits >= 64) ? (1 << 8) : 0)); | |
| const int expected_lane_sizes_mask = | |
| (lanes_per_largest_u8_vect >= (kMinBits / 8)) | |
| ? kGEVectSupportedLaneSizesMask | |
| : 0; | |
| constexpr size_t kSupportedU8VecSizesMask = | |
| static_cast<size_t>(((static_cast<size_t>(HWY_MAX_BYTES) << 1) - 1) & | |
| (~((kMinBits / 8) - 1))); | |
| HWY_ASSERT(TestForGEVectorsState.lane_sizes_mask == expected_lane_sizes_mask); | |
| #endif // HWY_TARGET == HWY_SCALAR | |
| #if HWY_HAVE_SCALABLE | |
| constexpr int kMinVecPow2 = | |
| static_cast<int>(CeilLog2(HWY_MIN(kMinBits / 16, 8))) - 3; | |
| static_assert(kMinVecPow2 >= -3 && kMinVecPow2 <= 0, | |
| "kMinVecPow2 must be between -3 and 0"); | |
| constexpr int kGEVectSupportedU8Pow2Mask = | |
| kSupportedU8Pow2Mask & (-(1 << (kMinVecPow2 + 3))); | |
| #if HWY_TARGET == HWY_RVV | |
| const int ge_vect_supported_u8_pow2_mask = | |
| kGEVectSupportedU8Pow2Mask & | |
| ((kMinBits <= 128) | |
| ? -1 | |
| : ((lanes_per_u8_vect < (kMinBits / 64)) | |
| ? 0 | |
| : (0x40 | | |
| ((lanes_per_u8_vect >= (kMinBits / 32)) ? 0x20 : 0) | | |
| ((lanes_per_u8_vect >= (kMinBits / 16)) ? 0x10 : 0) | | |
| ((lanes_per_u8_vect >= (kMinBits / 8)) ? 0x08 : 0)))); | |
| #else | |
| const int ge_vect_supported_u8_pow2_mask = | |
| (lanes_per_u8_vect >= (kMinBits / 8)) ? kGEVectSupportedU8Pow2Mask : 0; | |
| #endif | |
| #endif | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForGEVectorsState.per_lane_size_states[lane_size]; | |
| #if HWY_TARGET == HWY_SCALAR | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #else // HWY_TARGET != HWY_SCALAR | |
| if (kMinBits >= static_cast<size_t>(lane_size * 8)) { | |
| const size_t expected_lanes_mask = | |
| (((lanes_per_largest_u8_vect << 1) - 1) & kSupportedU8VecSizesMask) / | |
| static_cast<size_t>(lane_size); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == expected_lanes_mask); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| ge_vect_supported_u8_pow2_mask & (-((lane_size + 1) / 2)); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } else { | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #if HWY_HAVE_SCALABLE | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == 0); | |
| #endif | |
| } | |
| #endif // HWY_TARGET == HWY_SCALAR | |
| } | |
| } | |
| HWY_NOINLINE void TestAllForGEVectors() { | |
| DoTestAllForGEVectors<16>(ForGEVectors<16, TestForGEVectors>()); | |
| DoTestAllForGEVectors<32>(ForGEVectors<32, TestForGEVectors>()); | |
| DoTestAllForGEVectors<64>(ForGEVectors<64, TestForGEVectors>()); | |
| DoTestAllForGEVectors<128>(ForGEVectors<128, TestForGEVectors>()); | |
| DoTestAllForGEVectors<256>(ForGEVectors<256, TestForGEVectors>()); | |
| DoTestAllForGEVectors<512>(ForGEVectors<512, TestForGEVectors>()); | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForPromoteVectors) | |
| template <int kSrcLaneSizePow2, int kPromotePow2, class Test, class T, | |
| hwy::EnableIf<(kSrcLaneSizePow2 + kPromotePow2 <= | |
| (HWY_HAVE_INTEGER64 ? 3 : 2))> * = nullptr> | |
| static HWY_INLINE void ExecuteTestForPromoteVectors(const Test &test, | |
| T /*unused*/) { | |
| test(T()); | |
| } | |
| template <int kSrcLaneSizePow2, int kPromotePow2, class Test, class T, | |
| hwy::EnableIf<(kSrcLaneSizePow2 + kPromotePow2 > | |
| (HWY_HAVE_INTEGER64 ? 3 : 2))> * = nullptr> | |
| static HWY_INLINE void ExecuteTestForPromoteVectors(const Test & /*test*/, | |
| T /*unused*/) {} | |
| template <int kPow2> | |
| static HWY_NOINLINE void DoTestAllForPromoteVectors() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForPromoteVectorsState); | |
| const ForPromoteVectors<TestForPromoteVectors, kPow2> test; | |
| test(uint8_t()); | |
| ExecuteTestForPromoteVectors<1, kPow2>(test, uint16_t()); | |
| ExecuteTestForPromoteVectors<2, kPow2>(test, uint32_t()); | |
| constexpr int kMaxSupportedPromoteLaneSize = kMaxSupportedLaneSize >> kPow2; | |
| static_assert(kMaxSupportedPromoteLaneSize > 0, | |
| "kMaxSupportedPromoteLaneSize > 0 must be true"); | |
| constexpr int kSupportedPromoteLaneSizesMask = | |
| kSupportedLaneSizesMask & ((2 << kMaxSupportedPromoteLaneSize) - 1); | |
| HWY_ASSERT(TestForPromoteVectorsState.lane_sizes_mask == | |
| kSupportedPromoteLaneSizesMask); | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForPromoteVectorsState.per_lane_size_states[lane_size]; | |
| if (lane_size <= kMaxSupportedPromoteLaneSize) { | |
| const size_t lanes = | |
| LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size) >> | |
| (HWY_TARGET == HWY_SCALAR ? 0 : kPow2); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == | |
| ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1)); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| ((kSupportedU8Pow2Mask >> kPow2) & (-((lane_size + 1) / 2))) | | |
| ((HWY_TARGET == HWY_RVV) ? 0 : (1 << (3 - kPow2))); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } else { | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #if HWY_HAVE_SCALABLE | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == 0); | |
| #endif | |
| } | |
| } | |
| } | |
| HWY_NOINLINE void TestAllForPromoteVectors() { | |
| DoTestAllForPromoteVectors<1>(); | |
| DoTestAllForPromoteVectors<2>(); | |
| #if HWY_HAVE_INTEGER64 | |
| DoTestAllForPromoteVectors<3>(); | |
| #endif | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForDemoteVectors) | |
| template <int kSrcLaneSizePow2, int kDemotePow2, class Test, class T, | |
| hwy::EnableIf<(kSrcLaneSizePow2 >= kDemotePow2)> * = nullptr> | |
| static HWY_INLINE void ExecuteTestForDemoteVectors(const Test &test, | |
| T /*unused*/) { | |
| test(T()); | |
| } | |
| template <int kSrcLaneSizePow2, int kDemotePow2, class Test, class T, | |
| hwy::EnableIf<(kSrcLaneSizePow2 < kDemotePow2)> * = nullptr> | |
| static HWY_INLINE void ExecuteTestForDemoteVectors(const Test & /*test*/, | |
| T /*unused*/) {} | |
| template <int kPow2> | |
| HWY_NOINLINE void DoTestAllForDemoteVectors() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForDemoteVectorsState); | |
| const ForDemoteVectors<TestForDemoteVectors, kPow2> test; | |
| ExecuteTestForDemoteVectors<1, kPow2>(test, uint16_t()); | |
| ExecuteTestForDemoteVectors<2, kPow2>(test, uint32_t()); | |
| #if HWY_HAVE_INTEGER64 | |
| ExecuteTestForDemoteVectors<3, kPow2>(test, uint64_t()); | |
| #endif | |
| constexpr int kMinDemotableLaneSize = 1 << kPow2; | |
| constexpr int kSupportedDemoteLaneSizesMask = | |
| kSupportedLaneSizesMask & (-(1 << kMinDemotableLaneSize)); | |
| HWY_ASSERT(TestForDemoteVectorsState.lane_sizes_mask == | |
| kSupportedDemoteLaneSizesMask); | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForDemoteVectorsState.per_lane_size_states[lane_size]; | |
| if (lane_size >= kMinDemotableLaneSize) { | |
| const size_t lanes = | |
| LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == | |
| ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1)); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| kSupportedU8Pow2Mask & | |
| ((-lane_size) | (((lane_size >> kPow2) > 1) ? (lane_size >> 1) : 0)); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } else { | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0); | |
| #if HWY_HAVE_SCALABLE | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == 0); | |
| #endif | |
| } | |
| } | |
| } | |
| HWY_NOINLINE void TestAllForDemoteVectors() { | |
| DoTestAllForDemoteVectors<1>(); | |
| DoTestAllForDemoteVectors<2>(); | |
| #if HWY_HAVE_INTEGER64 | |
| DoTestAllForDemoteVectors<3>(); | |
| #endif | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForHalfVectors) | |
| template <int kPow2> | |
| static HWY_NOINLINE void DoTestAllForHalfVectors() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForHalfVectorsState); | |
| ForUnsignedTypes(ForHalfVectors<TestForHalfVectors, kPow2>()); | |
| #if HWY_TARGET == HWY_SCALAR | |
| const size_t kMinSrcVectLanes = 1; | |
| #else | |
| const size_t kMinSrcVectLanes = size_t{1} << kPow2; | |
| #endif | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| #if HWY_TARGET == HWY_SCALAR || HWY_TARGET == HWY_RVV | |
| const int expected_lane_sizes_mask = kSupportedLaneSizesMask; | |
| #else | |
| const int expected_lane_sizes_mask = | |
| kSupportedLaneSizesMask & | |
| (((lanes_per_u8_vect >= kMinSrcVectLanes) ? (1 << 1) : 0) | | |
| ((lanes_per_u8_vect >= 2 * kMinSrcVectLanes) ? (1 << 2) : 0) | | |
| ((lanes_per_u8_vect >= 4 * kMinSrcVectLanes) ? (1 << 4) : 0) | | |
| ((lanes_per_u8_vect >= 8 * kMinSrcVectLanes) ? (1 << 8) : 0)); | |
| #endif | |
| HWY_ASSERT(TestForHalfVectorsState.lane_sizes_mask == | |
| expected_lane_sizes_mask); | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForHalfVectorsState.per_lane_size_states[lane_size]; | |
| const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == | |
| (((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1) & | |
| (size_t{0} - kMinSrcVectLanes))); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| kSupportedU8Pow2Mask & ((-(((lane_size + 1) / 2) << kPow2)) | | |
| (lanes >= kMinSrcVectLanes ? 8 : 0)); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } | |
| } | |
| HWY_NOINLINE void TestAllForHalfVectors() { | |
| DoTestAllForHalfVectors<1>(); | |
| DoTestAllForHalfVectors<2>(); | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForPartialVectors) | |
| HWY_NOINLINE void TestAllForPartialVectors() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForPartialVectorsState); | |
| ForUnsignedTypes(ForPartialVectors<TestForPartialVectors>()); | |
| HWY_ASSERT(TestForPartialVectorsState.lane_sizes_mask == | |
| kSupportedLaneSizesMask); | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForPartialVectorsState.per_lane_size_states[lane_size]; | |
| const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == | |
| ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1)); | |
| #if HWY_HAVE_SCALABLE | |
| const int expected_pow2_mask = | |
| kSupportedU8Pow2Mask & (-((lane_size + 1) / 2)); | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif | |
| } | |
| } | |
| HWY_DECLARE_FOREACH_VECTOR_TEST(TestForPartialFixedOrFullVectors) | |
| HWY_NOINLINE void TestAllForPartialFixedOrFullVectors() { | |
| ZeroBytes<sizeof(ForeachVectorTestState)>( | |
| &TestForPartialFixedOrFullVectorsState); | |
| ForUnsignedTypes( | |
| ForPartialFixedOrFullScalableVectors<TestForPartialFixedOrFullVectors>()); | |
| HWY_ASSERT(TestForPartialFixedOrFullVectorsState.lane_sizes_mask == | |
| kSupportedLaneSizesMask); | |
| const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>()); | |
| for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) { | |
| ForeachVectorTestPerLaneSizeState *per_lane_size_state = | |
| &TestForPartialFixedOrFullVectorsState.per_lane_size_states[lane_size]; | |
| const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size); | |
| #if HWY_TARGET == HWY_RVV | |
| const size_t expected_lanes_mask = | |
| ((lanes * 16) - 1) & (size_t{0} - ((lanes_per_u8_vect + 7) / 8)); | |
| #elif HWY_HAVE_SCALABLE || HWY_TARGET_IS_SVE | |
| const size_t expected_lanes_mask = lanes; | |
| #else | |
| const size_t expected_lanes_mask = (lanes << 1) - 1; | |
| #endif | |
| HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == expected_lanes_mask); | |
| #if HWY_HAVE_SCALABLE | |
| #if HWY_TARGET == HWY_RVV | |
| const int expected_pow2_mask = kSupportedU8Pow2Mask & (-lane_size); | |
| #else | |
| const int expected_pow2_mask = 8; | |
| #endif | |
| HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask); | |
| #endif // HWY_HAVE_SCALABLE | |
| } | |
| } | |
| #undef HWY_DECLARE_FOREACH_VECTOR_TEST | |
| } // namespace | |
| // NOLINTNEXTLINE(google-readability-namespace-comments) | |
| } // namespace HWY_NAMESPACE | |
| } // namespace hwy | |
| HWY_AFTER_NAMESPACE(); | |
| #if HWY_ONCE | |
| namespace hwy { | |
| namespace { | |
| HWY_BEFORE_TEST(HwyForeachVecTest); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForMaxPow2); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForExtendableVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForShrinkableVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForGEVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForPromoteVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForDemoteVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForHalfVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForPartialVectors); | |
| HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForPartialFixedOrFullVectors); | |
| HWY_AFTER_TEST(); | |
| } // namespace | |
| } // namespace hwy | |
| HWY_TEST_MAIN(); | |
| #endif // HWY_ONCE |