| // Copyright 2019 Google LLC | |
| // SPDX-License-Identifier: Apache-2.0 | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #undef HWY_TARGET_INCLUDE | |
| #define HWY_TARGET_INCLUDE "tests/shift_test.cc" | |
| #include "hwy/foreach_target.h" // IWYU pragma: keep | |
| #include "hwy/highway.h" | |
| #include "hwy/tests/test_util-inl.h" | |
| HWY_BEFORE_NAMESPACE(); | |
| namespace hwy { | |
| namespace HWY_NAMESPACE { | |
| namespace { | |
| template <bool kSigned> | |
| struct TestLeftShifts { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T t, D d) { | |
| if (kSigned) { | |
| // Also test positive values | |
| TestLeftShifts</*kSigned=*/false>()(t, d); | |
| } | |
| using TI = MakeSigned<T>; | |
| using TU = MakeUnsigned<T>; | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| // Values to shift | |
| const Vec<D> values = Iota(d, kSigned ? -TI(N) : TI(0)); | |
| constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; | |
| // 0 | |
| HWY_ASSERT_VEC_EQ(d, values, ShiftLeft<0>(values)); | |
| HWY_ASSERT_VEC_EQ(d, values, ShiftLeftSame(values, 0)); | |
| // 1 | |
| for (size_t i = 0; i < N; ++i) { | |
| const T value = | |
| kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N)) | |
| : static_cast<T>(i); | |
| expected[i] = static_cast<T>(static_cast<TU>(value) << 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeft<1>(values)); | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftSame(values, 1)); | |
| // max | |
| for (size_t i = 0; i < N; ++i) { | |
| const T value = | |
| kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N)) | |
| : static_cast<T>(i); | |
| expected[i] = static_cast<T>(static_cast<TU>(value) << kMaxShift); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeft<kMaxShift>(values)); | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftSame(values, kMaxShift)); | |
| } | |
| }; | |
| template <bool kSigned> | |
| struct TestVariableLeftShifts { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T t, D d) { | |
| if (kSigned) { | |
| // Also test positive values | |
| TestVariableLeftShifts</*kSigned=*/false>()(t, d); | |
| } | |
| using TI = MakeSigned<T>; | |
| using TU = MakeUnsigned<T>; | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> v1 = Set(d, 1); | |
| const Vec<D> values = Iota(d, kSigned ? -TI(N) : TI(0)); // value to shift | |
| constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; | |
| const Vec<D> max_shift = Set(d, kMaxShift); | |
| const Vec<D> small_shifts = And(Iota(d, 0), max_shift); | |
| const Vec<D> large_shifts = Sub(max_shift, small_shifts); | |
| // Same: 0 | |
| HWY_ASSERT_VEC_EQ(d, values, Shl(values, v0)); | |
| // Same: 1 | |
| for (size_t i = 0; i < N; ++i) { | |
| const T value = | |
| kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N)) | |
| : static_cast<T>(i); | |
| expected[i] = static_cast<T>(static_cast<TU>(value) << 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, v1)); | |
| // Same: max | |
| for (size_t i = 0; i < N; ++i) { | |
| const T value = | |
| kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N)) | |
| : static_cast<T>(i); | |
| expected[i] = static_cast<T>(static_cast<TU>(value) << kMaxShift); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, max_shift)); | |
| // Variable: small | |
| for (size_t i = 0; i < N; ++i) { | |
| const T value = | |
| kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N)) | |
| : static_cast<T>(i); | |
| expected[i] = static_cast<T>(static_cast<TU>(value) << (i & kMaxShift)); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, small_shifts)); | |
| // Variable: large | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = | |
| static_cast<T>(static_cast<TU>(1) << (kMaxShift - (i & kMaxShift))); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(v1, large_shifts)); | |
| } | |
| }; | |
| struct TestUnsignedRightShifts { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| const Vec<D> values = Iota(d, 0); | |
| const T kMax = LimitsMax<T>(); | |
| constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; | |
| // Shift by 0 | |
| HWY_ASSERT_VEC_EQ(d, values, ShiftRight<0>(values)); | |
| HWY_ASSERT_VEC_EQ(d, values, ShiftRightSame(values, 0)); | |
| // Shift by 1 | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(values)); | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, 1)); | |
| // max | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> kMaxShift); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<kMaxShift>(values)); | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, kMaxShift)); | |
| } | |
| }; | |
| struct TestVariableUnsignedRightShifts { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> v1 = Set(d, 1); | |
| const Vec<D> values = Iota(d, 0); | |
| const T kMax = LimitsMax<T>(); | |
| const Vec<D> max = Set(d, kMax); | |
| constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; | |
| const Vec<D> max_shift = Set(d, kMaxShift); | |
| const Vec<D> small_shifts = And(Iota(d, 0), max_shift); | |
| const Vec<D> large_shifts = Sub(max_shift, small_shifts); | |
| // Same: 0 | |
| HWY_ASSERT_VEC_EQ(d, values, Shr(values, v0)); | |
| // Same: 1 | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, v1)); | |
| // Same: max | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> kMaxShift); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, max_shift)); | |
| // Variable: small | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = ConvertScalarTo<T>(T(i) >> (i & kMaxShift)); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, small_shifts)); | |
| // Variable: Large | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = ConvertScalarTo<T>(kMax >> (kMaxShift - (i & kMaxShift))); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(max, large_shifts)); | |
| } | |
| }; | |
| template <int kAmount, typename T> | |
| T RightShiftNegative(T val) { | |
| // C++ shifts are implementation-defined for negative numbers, and we have | |
| // seen divisions replaced with shifts, so resort to bit operations. | |
| using TU = hwy::MakeUnsigned<T>; | |
| TU bits; | |
| CopySameSize(&val, &bits); | |
| const TU shifted = TU(bits >> kAmount); | |
| const TU all = TU(~TU(0)); | |
| const size_t num_zero = sizeof(TU) * 8 - 1 - kAmount; | |
| const TU sign_extended = static_cast<TU>((all << num_zero) & LimitsMax<TU>()); | |
| bits = shifted | sign_extended; | |
| CopySameSize(&bits, &val); | |
| return val; | |
| } | |
| class TestSignedRightShifts { | |
| public: | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| constexpr T kMin = LimitsMin<T>(); | |
| constexpr T kMax = LimitsMax<T>(); | |
| constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; | |
| // First test positive values, negative are checked below. | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> values = And(Iota(d, 0), Set(d, kMax)); | |
| // Shift by 0 | |
| HWY_ASSERT_VEC_EQ(d, values, ShiftRight<0>(values)); | |
| HWY_ASSERT_VEC_EQ(d, values, ShiftRightSame(values, 0)); | |
| // Shift by 1 | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(values)); | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, 1)); | |
| // max | |
| HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<kMaxShift>(values)); | |
| HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(values, kMaxShift)); | |
| // Even negative value | |
| Test<0>(kMin, d, __LINE__); | |
| Test<1>(kMin, d, __LINE__); | |
| Test<2>(kMin, d, __LINE__); | |
| Test<kMaxShift>(kMin, d, __LINE__); | |
| const T odd = ConvertScalarTo<T>(kMin + 1); | |
| Test<0>(odd, d, __LINE__); | |
| Test<1>(odd, d, __LINE__); | |
| Test<2>(odd, d, __LINE__); | |
| Test<kMaxShift>(odd, d, __LINE__); | |
| } | |
| private: | |
| template <int kAmount, typename T, class D> | |
| void Test(T val, D d, int line) { | |
| const Vec<D> expected = Set(d, RightShiftNegative<kAmount>(val)); | |
| const Vec<D> in = Set(d, val); | |
| const char* file = __FILE__; | |
| AssertVecEqual(d, expected, ShiftRight<kAmount>(in), file, line); | |
| AssertVecEqual(d, expected, ShiftRightSame(in, kAmount), file, line); | |
| } | |
| }; | |
| struct TestVariableSignedRightShifts { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| using TU = MakeUnsigned<T>; | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| constexpr T kMin = LimitsMin<T>(); | |
| constexpr T kMax = LimitsMax<T>(); | |
| constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; | |
| // First test positive values, negative are checked below. | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> positive = And(Iota(d, 0), Set(d, kMax)); | |
| // Shift by 0 | |
| HWY_ASSERT_VEC_EQ(d, positive, ShiftRight<0>(positive)); | |
| HWY_ASSERT_VEC_EQ(d, positive, ShiftRightSame(positive, 0)); | |
| // Shift by 1 | |
| for (size_t i = 0; i < N; ++i) { | |
| expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(positive)); | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(positive, 1)); | |
| // max | |
| HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<kMaxShift>(positive)); | |
| HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(positive, kMaxShift)); | |
| const Vec<D> max_shift = Set(d, kMaxShift); | |
| const Vec<D> small_shifts = And(Iota(d, 0), max_shift); | |
| const Vec<D> large_shifts = Sub(max_shift, small_shifts); | |
| const Vec<D> negative = Iota(d, kMin); | |
| // Test varying negative to shift | |
| for (size_t i = 0; i < N; ++i) { | |
| const T val = ConvertScalarTo<T>(static_cast<TU>(kMin) + i); | |
| expected[i] = | |
| (val < 0) ? RightShiftNegative<1>(val) : ConvertScalarTo<T>(val >> 1); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(negative, Set(d, 1))); | |
| // Shift MSB right by small amounts | |
| for (size_t i = 0; i < N; ++i) { | |
| const size_t amount = i & kMaxShift; | |
| const TU shifted = static_cast<TU>(~((1ull << (kMaxShift - amount)) - 1)); | |
| CopySameSize(&shifted, &expected[i]); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(Set(d, kMin), small_shifts)); | |
| // Shift MSB right by large amounts | |
| for (size_t i = 0; i < N; ++i) { | |
| const size_t amount = kMaxShift - (i & kMaxShift); | |
| const TU shifted = static_cast<TU>(~((1ull << (kMaxShift - amount)) - 1)); | |
| CopySameSize(&shifted, &expected[i]); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(Set(d, kMin), large_shifts)); | |
| } | |
| }; | |
| HWY_NOINLINE void TestAllShifts() { | |
| ForUnsignedTypes(ForPartialVectors<TestLeftShifts</*kSigned=*/false>>()); | |
| ForSignedTypes(ForPartialVectors<TestLeftShifts</*kSigned=*/true>>()); | |
| ForUnsignedTypes(ForPartialVectors<TestUnsignedRightShifts>()); | |
| ForSignedTypes(ForPartialVectors<TestSignedRightShifts>()); | |
| } | |
| HWY_NOINLINE void TestAllVariableShifts() { | |
| ForUnsignedTypes( | |
| ForPartialVectors<TestVariableLeftShifts</*kSigned=*/false>>()); | |
| ForSignedTypes(ForPartialVectors<TestVariableLeftShifts</*kSigned=*/true>>()); | |
| ForUnsignedTypes(ForPartialVectors<TestVariableUnsignedRightShifts>()); | |
| ForSignedTypes(ForPartialVectors<TestVariableSignedRightShifts>()); | |
| } | |
| struct TestRoundingShiftRight { | |
| template <int kShiftAmt, class D> | |
| static HWY_INLINE void VerifyRoundingShiftRight( | |
| D d, const TFromD<D>* HWY_RESTRICT expected, Vec<D> v, | |
| const char* filename, const int line) { | |
| AssertVecEqual(d, expected, RoundingShiftRight<kShiftAmt>(v), filename, | |
| line); | |
| AssertVecEqual(d, expected, RoundingShiftRightSame(v, kShiftAmt), filename, | |
| line); | |
| } | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| using TU = MakeUnsigned<T>; | |
| const Vec<D> iota0 = Iota(d, T{0}); | |
| const size_t N = Lanes(d); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| Store(iota0, d, expected.get()); | |
| VerifyRoundingShiftRight<0>(d, expected.get(), iota0, __FILE__, __LINE__); | |
| const Vec<D> v1 = Set(d, T{1}); | |
| const Vec<D> iota1 = Add(iota0, v1); | |
| Store(iota1, d, expected.get()); | |
| VerifyRoundingShiftRight<0>(d, expected.get(), iota1, __FILE__, __LINE__); | |
| const Vec<D> v2 = Set(d, T{2}); | |
| const Vec<D> iota2 = Add(iota0, v2); | |
| Store(iota2, d, expected.get()); | |
| VerifyRoundingShiftRight<0>(d, expected.get(), iota2, __FILE__, __LINE__); | |
| const Vec<D> iota3 = Add(iota0, Set(d, T{3})); | |
| Store(iota3, d, expected.get()); | |
| VerifyRoundingShiftRight<0>(d, expected.get(), iota3, __FILE__, __LINE__); | |
| const Vec<D> seq4 = Add(iota0, SignBit(d)); | |
| Store(seq4, d, expected.get()); | |
| VerifyRoundingShiftRight<0>(d, expected.get(), seq4, __FILE__, __LINE__); | |
| const Vec<D> seq5 = Add(seq4, v1); | |
| Store(seq5, d, expected.get()); | |
| VerifyRoundingShiftRight<0>(d, expected.get(), seq5, __FILE__, __LINE__); | |
| const Vec<D> v0 = Zero(d); | |
| Store(AverageRound(iota1, v0), d, expected.get()); | |
| VerifyRoundingShiftRight<1>(d, expected.get(), iota1, __FILE__, __LINE__); | |
| Store(AverageRound(iota2, v0), d, expected.get()); | |
| VerifyRoundingShiftRight<1>(d, expected.get(), iota2, __FILE__, __LINE__); | |
| Store(AverageRound(seq4, v0), d, expected.get()); | |
| VerifyRoundingShiftRight<1>(d, expected.get(), seq4, __FILE__, __LINE__); | |
| Store(AverageRound(seq5, v0), d, expected.get()); | |
| VerifyRoundingShiftRight<1>(d, expected.get(), seq5, __FILE__, __LINE__); | |
| const Vec<D> seq6 = And( | |
| Xor(iota1, | |
| Set(d, static_cast<T>(0x70FB991A05AC6B24ULL & LimitsMax<TU>()))), | |
| Set(d, static_cast<T>(~static_cast<T>(0x10)))); | |
| Store(ShiftRight<5>(seq6), d, expected.get()); | |
| VerifyRoundingShiftRight<5>(d, expected.get(), seq6, __FILE__, __LINE__); | |
| const Vec<D> seq7 = | |
| Or(Xor(iota2, | |
| Set(d, static_cast<T>(0x6ED498B16EC87C63ULL & LimitsMax<TU>()))), | |
| Set(d, static_cast<T>(0x04))); | |
| Store(Add(ShiftRight<3>(seq7), v1), d, expected.get()); | |
| VerifyRoundingShiftRight<3>(d, expected.get(), seq7, __FILE__, __LINE__); | |
| const Vec<D> seq8 = And( | |
| Xor(iota1, | |
| Set(d, static_cast<T>(0x186958FE04C94D77ULL & LimitsMax<TU>()))), | |
| Set(d, static_cast<T>(~static_cast<T>(0x08)))); | |
| Store(ShiftRight<4>(seq8), d, expected.get()); | |
| VerifyRoundingShiftRight<4>(d, expected.get(), seq8, __FILE__, __LINE__); | |
| const Vec<D> seq9 = | |
| Or(Xor(iota2, | |
| Set(d, static_cast<T>(0x7FC4E62077CC7655ULL & LimitsMax<TU>()))), | |
| v2); | |
| Store(Add(ShiftRight<2>(seq9), v1), d, expected.get()); | |
| VerifyRoundingShiftRight<2>(d, expected.get(), seq9, __FILE__, __LINE__); | |
| } | |
| }; | |
| HWY_NOINLINE void TestAllRoundingShiftRight() { | |
| ForIntegerTypes(ForPartialVectors<TestRoundingShiftRight>()); | |
| } | |
| template <typename ToT> | |
| struct TestShiftRightAndDemoteTo { | |
| template <int kShiftAmt, class DTo, class V> | |
| static HWY_INLINE void Verify(DTo to_d, V in, const char* filename, | |
| const int line) { | |
| AssertVecEqual(to_d, DemoteTo(to_d, ShiftRight<kShiftAmt>(in)), | |
| ShiftRightAndDemoteTo<kShiftAmt>(to_d, in), filename, line); | |
| AssertVecEqual(to_d, DemoteTo(to_d, RoundingShiftRight<kShiftAmt>(in)), | |
| RoundingShiftRightAndDemoteTo<kShiftAmt>(to_d, in), filename, | |
| line); | |
| } | |
| template <typename T, class DFrom> | |
| HWY_NOINLINE void operator()(T /*unused*/, DFrom from_d) { | |
| static_assert(sizeof(T) > sizeof(ToT), "Input type must be wider"); | |
| const Rebind<ToT, DFrom> to_d; | |
| using V = Vec<DFrom>; | |
| using TU = MakeUnsigned<T>; | |
| const V iota1 = Add(Iota(from_d, T{0}), Set(from_d, T{1})); | |
| const V seq_neg = Add(iota1, SignBit(from_d)); | |
| const V seq_sat = | |
| Or(Xor(iota1, Set(from_d, static_cast<T>(0x6ED498B16EC87C63ULL & | |
| LimitsMax<TU>()))), | |
| Set(from_d, T{1})); | |
| Verify<0>(to_d, iota1, __FILE__, __LINE__); | |
| Verify<0>(to_d, seq_neg, __FILE__, __LINE__); | |
| Verify<1>(to_d, iota1, __FILE__, __LINE__); | |
| Verify<1>(to_d, seq_neg, __FILE__, __LINE__); | |
| Verify<1>(to_d, seq_sat, __FILE__, __LINE__); | |
| Verify<static_cast<int>(sizeof(ToT) * 4)>(to_d, seq_neg, __FILE__, | |
| __LINE__); | |
| Verify<static_cast<int>(sizeof(ToT) * 8 - 1)>(to_d, seq_sat, __FILE__, | |
| __LINE__); | |
| Verify<static_cast<int>(sizeof(ToT) * 8)>(to_d, seq_sat, __FILE__, | |
| __LINE__); | |
| Verify<static_cast<int>(sizeof(T) * 8 - 1)>(to_d, seq_neg, __FILE__, | |
| __LINE__); | |
| } | |
| }; | |
| HWY_NOINLINE void TestAllShiftRightAndDemoteTo() { | |
| const ForDemoteVectors<TestShiftRightAndDemoteTo<uint8_t>> from_i16_to_u8; | |
| from_i16_to_u8(int16_t()); | |
| from_i16_to_u8(uint16_t()); | |
| const ForDemoteVectors<TestShiftRightAndDemoteTo<int8_t>> from_i16_to_i8; | |
| from_i16_to_i8(int16_t()); | |
| from_i16_to_i8(uint16_t()); | |
| const ForDemoteVectors<TestShiftRightAndDemoteTo<uint16_t>> from_i32_to_u16; | |
| from_i32_to_u16(int32_t()); | |
| from_i32_to_u16(uint32_t()); | |
| const ForDemoteVectors<TestShiftRightAndDemoteTo<int16_t>> from_i32_to_i16; | |
| from_i32_to_i16(int32_t()); | |
| from_i32_to_i16(uint32_t()); | |
| #if HWY_HAVE_INTEGER64 | |
| const ForDemoteVectors<TestShiftRightAndDemoteTo<uint32_t>> from_i64_to_u32; | |
| from_i64_to_u32(int64_t()); | |
| from_i64_to_u32(uint64_t()); | |
| const ForDemoteVectors<TestShiftRightAndDemoteTo<int32_t>> from_i64_to_i32; | |
| from_i64_to_i32(int64_t()); | |
| from_i64_to_i32(uint64_t()); | |
| #endif | |
| } | |
| struct TestVariableRoundingShr { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const size_t N = Lanes(d); | |
| constexpr size_t kNumOfBits = sizeof(T) * 8; | |
| const Vec<D> iota1 = Iota(d, T{1}); | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> v1 = Set(d, T{1}); | |
| const Vec<D> sign_bit = SignBit(d); | |
| for (size_t i = 0; i < kNumOfBits; i += N) { | |
| Vec<D> shift_amt = Iota(d, static_cast<T>(i & (kNumOfBits - 1))); | |
| HWY_IF_CONSTEXPR(HWY_MAX_LANES_D(D) > kNumOfBits) { | |
| shift_amt = And(shift_amt, Set(d, static_cast<T>(kNumOfBits - 1))); | |
| } | |
| const Vec<D> half_bit = ShiftRight<1>(Shl(v1, shift_amt)); | |
| const Vec<D> in_0 = AndNot(half_bit, Or(Shl(iota1, shift_amt), v1)); | |
| const Vec<D> in_1 = Or(in_0, half_bit); | |
| const Vec<D> in_2 = Xor(in_0, sign_bit); | |
| const Vec<D> in_3 = Xor(in_1, sign_bit); | |
| const Vec<D> round_decr = VecFromMask(d, Ne(half_bit, v0)); | |
| const Vec<D> expected_0 = Shr(in_0, shift_amt); | |
| const Vec<D> expected_1 = Sub(Shr(in_1, shift_amt), round_decr); | |
| const Vec<D> expected_2 = Shr(in_2, shift_amt); | |
| const Vec<D> expected_3 = Sub(Shr(in_3, shift_amt), round_decr); | |
| HWY_ASSERT_VEC_EQ(d, expected_0, RoundingShr(in_0, shift_amt)); | |
| HWY_ASSERT_VEC_EQ(d, expected_1, RoundingShr(in_1, shift_amt)); | |
| HWY_ASSERT_VEC_EQ(d, expected_2, RoundingShr(in_2, shift_amt)); | |
| HWY_ASSERT_VEC_EQ(d, expected_3, RoundingShr(in_3, shift_amt)); | |
| } | |
| } | |
| }; | |
| HWY_NOINLINE void TestAllVariableRoundingShr() { | |
| ForIntegerTypes(ForPartialVectors<TestVariableRoundingShr>()); | |
| } | |
| struct TestMaskedShiftLeftOr { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const Vec<D> v1 = Iota(d, 1); | |
| const Vec<D> v2 = Iota(d, 2); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| const Vec<D> expected = IfThenElse(first_five, ShiftLeft<1>(v2), v1); | |
| HWY_ASSERT_VEC_EQ(d, expected, MaskedShiftLeftOr<1>(v1, first_five, v2)); | |
| } | |
| }; | |
| struct TestMaskedShiftLeft { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const Mask<D> all_true = MaskTrue(d); | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> v1 = Iota(d, 1); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| HWY_ASSERT_VEC_EQ(d, ShiftLeft<1>(v1), MaskedShiftLeft<1>(all_true, v1)); | |
| const Vec<D> v1_exp = IfThenElse(first_five, ShiftLeft<1>(v1), v0); | |
| HWY_ASSERT_VEC_EQ(d, v1_exp, MaskedShiftLeft<1>(first_five, v1)); | |
| } | |
| }; | |
| struct TestMaskedShiftRightOr { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| // Test MaskedShiftRightOr | |
| const Vec<D> v1 = Iota(d, 1); | |
| const Vec<D> v2 = Iota(d, 2); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| const Vec<D> expected = IfThenElse(first_five, ShiftRight<1>(v2), v1); | |
| HWY_ASSERT_VEC_EQ(d, expected, MaskedShiftRightOr<1>(v1, first_five, v2)); | |
| } | |
| }; | |
| struct TestMaskedShiftRight { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| const Mask<D> all_true = MaskTrue(d); | |
| const Vec<D> v0 = Zero(d); | |
| const Vec<D> v1 = Iota(d, 1); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| HWY_ASSERT_VEC_EQ(d, ShiftRight<1>(v1), MaskedShiftRight<1>(all_true, v1)); | |
| const Vec<D> v1_exp = IfThenElse(first_five, ShiftRight<1>(v1), v0); | |
| HWY_ASSERT_VEC_EQ(d, v1_exp, MaskedShiftRight<1>(first_five, v1)); | |
| } | |
| }; | |
| struct TestMaskedShlOr { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| // Test MaskedShlOr | |
| const Vec<D> v1 = Iota(d, 1); | |
| const Vec<D> v2 = Iota(d, 2); | |
| const Vec<D> shifts = Set(d, 1); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| const Vec<D> expected = IfThenElse(first_five, ShiftLeft<1>(v2), v1); | |
| HWY_ASSERT_VEC_EQ(d, expected, MaskedShlOr(v1, first_five, v2, shifts)); | |
| } | |
| }; | |
| struct TestMaskedShl { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| // Test MaskedShl | |
| const Vec<D> v2 = Iota(d, 2); | |
| const Vec<D> shifts = Set(d, 1); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| const Vec<D> expected = IfThenElseZero(first_five, ShiftLeft<1>(v2)); | |
| HWY_ASSERT_VEC_EQ(d, expected, MaskedShl(first_five, v2, shifts)); | |
| } | |
| }; | |
| struct TestMaskedShrOr { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| // Test MaskedShrOr | |
| const Vec<D> v1 = Iota(d, 1); | |
| const Vec<D> v2 = Iota(d, 2); | |
| const Vec<D> shifts = Set(d, 1); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| const Vec<D> expected = IfThenElse(first_five, ShiftRight<1>(v2), v1); | |
| HWY_ASSERT_VEC_EQ(d, expected, MaskedShrOr(v1, first_five, v2, shifts)); | |
| } | |
| }; | |
| struct TestMaskedShr { | |
| template <typename T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| // Test MaskedShr | |
| const Vec<D> v2 = Iota(d, 2); | |
| const Vec<D> shifts = Set(d, 1); | |
| const Mask<D> first_five = FirstN(d, 5); | |
| const Vec<D> expected = IfThenElseZero(first_five, ShiftRight<1>(v2)); | |
| HWY_ASSERT_VEC_EQ(d, expected, MaskedShr(first_five, v2, shifts)); | |
| } | |
| }; | |
| HWY_NOINLINE void TestAllMaskedShift() { | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShiftLeftOr>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShiftLeft>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShiftRightOr>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShiftRight>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShlOr>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShl>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShrOr>()); | |
| ForIntegerTypes(ForPartialVectors<TestMaskedShr>()); | |
| } | |
| struct TestMultiRotateRight { | |
| #if HWY_TARGET != HWY_SCALAR | |
| static HWY_INLINE uint64_t ByteSwapU64(uint64_t x) { | |
| // Equivalent to std::byteswap for uint64 (in C++23) | |
| return ((x << 56) & 0xff00000000000000ULL) | | |
| ((x << 40) & 0x00ff000000000000ULL) | | |
| ((x << 24) & 0x0000ff0000000000ULL) | | |
| ((x << 8) & 0x000000ff00000000ULL) | | |
| ((x >> 8) & 0x00000000ff000000ULL) | | |
| ((x >> 24) & 0x0000000000ff0000ULL) | | |
| ((x >> 40) & 0x000000000000ff00ULL) | | |
| ((x >> 56) & 0x00000000000000ffULL); | |
| } | |
| template <class D> | |
| static HWY_NOINLINE void DoTestMultiRotateRight(D d) { | |
| using T = TFromD<D>; | |
| using TU = MakeUnsigned<T>; | |
| const size_t N = Lanes(d); | |
| const Repartition<uint8_t, decltype(d)> du8; | |
| HWY_ALIGN static constexpr uint64_t kTestCases[16] = { | |
| 0x3FCF738F5342BA34ULL, 0x1643B72135F9C1ECULL, 0x9569D89F85EC5689ULL, | |
| 0x5C82AE2EE1291AF9ULL, 0x0BE183E0146F82D5ULL, 0xE264FB25B7FCAC70ULL, | |
| 0x16DCC74C5615DB8FULL, 0x5C6696FF3CF2910FULL, 0x0102030405060708ULL, | |
| 0x1020304050607080ULL, 0x0102010201020102ULL, 0x2010011002002012ULL, | |
| 0xF0E1D2C3B4A59687ULL, 0x78695A4B3C2D1E0FULL, 0x627BD89345E1CAF0ULL, | |
| 0x7BCA6F13D02945E8ULL}; | |
| HWY_ALIGN static constexpr uint8_t kRotateAmounts[128] = { | |
| 43, 15, 17, 29, 10, 16, 7, 45, 50, 3, 5, 52, 23, 6, 35, 25, | |
| 36, 24, 16, 6, 59, 61, 57, 20, 49, 5, 14, 11, 55, 32, 0, 54, | |
| 48, 44, 9, 28, 20, 28, 42, 60, 62, 12, 34, 63, 24, 27, 19, 46, | |
| 33, 19, 51, 47, 26, 31, 48, 61, 22, 4, 17, 40, 50, 53, 46, 34, | |
| 43, 22, 56, 39, 30, 18, 54, 40, 38, 36, 58, 9, 18, 38, 63, 41, | |
| 35, 0, 2, 13, 26, 41, 49, 3, 10, 1, 44, 21, 2, 7, 4, 57, | |
| 37, 13, 21, 29, 39, 52, 31, 37, 12, 58, 15, 8, 55, 33, 32, 59, | |
| 1, 53, 60, 8, 11, 23, 47, 30, 62, 51, 27, 56, 14, 42, 25, 45}; | |
| // Replicate the test cases into the padded vector, repeating as necessary | |
| // to fill the vector. | |
| const size_t padded = RoundUpTo(N, 16); | |
| auto in_lanes = AllocateAligned<T>(padded); | |
| auto in_shift_amounts = AllocateAligned<uint8_t>(padded * sizeof(T)); | |
| auto expected = AllocateAligned<T>(padded); | |
| HWY_ASSERT(in_lanes && in_shift_amounts && expected); | |
| for (size_t i = 0; i < padded; i += 16) { | |
| const size_t kBytes = 16 * sizeof(T); | |
| CopyBytes(kTestCases, in_lanes.get() + i, kBytes); | |
| CopyBytes(kRotateAmounts, in_shift_amounts.get() + i * sizeof(T), kBytes); | |
| } | |
| constexpr int kShiftAmtMask = static_cast<int>(sizeof(T) * 8 - 1); | |
| for (size_t i = 0; i < padded; i++) { | |
| const uint64_t input_val = | |
| static_cast<uint64_t>(static_cast<TU>(in_lanes[i])); | |
| for (size_t j = 0; j < sizeof(T); j++) { | |
| const size_t byte_idx = i * sizeof(T) + j; | |
| const int rotate_right_amt = | |
| static_cast<int>(in_shift_amounts[byte_idx]) & kShiftAmtMask; | |
| const uint8_t rotated_val = static_cast<uint8_t>( | |
| ((input_val >> rotate_right_amt) | | |
| (input_val << ((-rotate_right_amt) & kShiftAmtMask))) & | |
| 0xFF); | |
| CopyBytes(&rotated_val, | |
| reinterpret_cast<uint8_t*>(expected.get()) + byte_idx, | |
| sizeof(uint8_t)); | |
| } | |
| } | |
| for (size_t i = 0; i < padded; i += N) { | |
| HWY_ASSERT_VEC_EQ( | |
| d, expected.get() + i, | |
| MultiRotateRight(Load(d, in_lanes.get() + i), | |
| Load(du8, in_shift_amounts.get() + i * sizeof(T)))); | |
| } | |
| } | |
| template <class D, HWY_IF_LANES_D(D, 1)> | |
| static HWY_INLINE void DoGe128BitMultiRotateRightTests(D /*d*/) {} | |
| template <class D, HWY_IF_LANES_GT_D(D, 1)> | |
| static HWY_NOINLINE void DoGe128BitMultiRotateRightTests(D d) { | |
| using T = TFromD<D>; | |
| const Repartition<uint8_t, decltype(d)> du8; | |
| const size_t N = Lanes(d); | |
| if (N < 2) return; | |
| // Generate a vector where all bytes in a block are different | |
| const uint64_t initial_even = 0x0102030405060708ul; | |
| const uint64_t initial_odd = 0x1020304050607080ul; | |
| const Vec<D> v1 = Dup128VecFromValues(d, initial_even, initial_odd); | |
| // Test byte aligned shifts | |
| // First 8 values define the transformation for even lanes | |
| // Second 8 values define the transformation for odd lanes | |
| auto indices = | |
| Dup128VecFromValues(du8, 0, 8, 16, 24, 32, 40, 48, | |
| 56, // Return every byte to its original location | |
| 56, 48, 40, 32, 24, 16, 8, 0 // Reverse byte order | |
| ); | |
| auto expected = AllocateAligned<T>(N); | |
| HWY_ASSERT(expected); | |
| for (size_t i = 0; i < N; i += 2) { | |
| #if HWY_IS_LITTLE_ENDIAN | |
| expected[i] = ConvertScalarTo<T>(initial_even); | |
| expected[i + 1] = ConvertScalarTo<T>(ByteSwapU64(initial_odd)); | |
| #else | |
| expected[i] = ConvertScalarTo<T>(ByteSwapU64(initial_even)); | |
| expected[i + 1] = ConvertScalarTo<T>(initial_odd); | |
| #endif | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), MultiRotateRight(v1, indices)); | |
| // Test bit level shifts, different amounts for each byte | |
| const Vec<D> v2 = Set(d, 0x0102010201020102ul); | |
| indices = Dup128VecFromValues( | |
| du8, // Let j = i % 8 | |
| 0, 9, 18, 27, 36, 45, 54, | |
| 63, // Shifting right: r[i].byte[j] = (v[j..j+1] >> j) & 0xff | |
| 0, 7, 14, 21, 28, 35, 42, | |
| 49 // Shifting left: r[i].byte[j] = ((v[j-1..j] << j) >> 8) & 0xff | |
| ); | |
| for (size_t i = 0; i < N; ++i) { | |
| const T v_i = ExtractLane(v2, i); | |
| T shift_result = 0; | |
| for (size_t j = 0; j < 8; j++) { | |
| int idx = static_cast<int>(ExtractLane(indices, (i * 8) + j)); | |
| T rot_result = | |
| ConvertScalarTo<T>((static_cast<uint64_t>(v_i) >> idx) | | |
| (static_cast<uint64_t>(v_i) << ((-idx) & 63))); | |
| #if HWY_IS_LITTLE_ENDIAN | |
| shift_result |= (rot_result & 0xff) << (j * 8); | |
| #else | |
| shift_result |= (rot_result & 0xff) << ((j ^ 7) * 8); | |
| #endif | |
| } | |
| expected[i] = ConvertScalarTo<T>(shift_result); | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), MultiRotateRight(v2, indices)); | |
| // Combine byte-level reordering with bit level shift | |
| indices = Dup128VecFromValues( | |
| du8, 4, 12, 20, 28, 36, 44, 52, 60, // Shift each byte right 4 bits | |
| 60, 52, 44, 36, 28, 20, 12, | |
| 4 // Shift each byte right 4 bits then reverse byte order | |
| ); | |
| for (size_t i = 0; i < N; i += 2) { | |
| const uint64_t shifted_initial_even = | |
| (initial_even >> 4) | (initial_even << (64 - 4)); | |
| #if HWY_IS_LITTLE_ENDIAN | |
| expected[i] = ConvertScalarTo<T>(shifted_initial_even); | |
| #else | |
| expected[i] = ConvertScalarTo<T>(ByteSwapU64(shifted_initial_even)); | |
| #endif | |
| uint64_t unreversed_val = | |
| ConvertScalarTo<T>((initial_odd >> 4) | (initial_odd << (64 - 4))); | |
| #if HWY_IS_LITTLE_ENDIAN | |
| expected[i + 1] = ConvertScalarTo<T>(ByteSwapU64(unreversed_val)); | |
| #else | |
| expected[i + 1] = ConvertScalarTo<T>(unreversed_val); | |
| #endif | |
| } | |
| HWY_ASSERT_VEC_EQ(d, expected.get(), MultiRotateRight(v1, indices)); | |
| } | |
| #endif // HWY_TARGET != HWY_SCALAR | |
| template <class T, class D> | |
| HWY_NOINLINE void operator()(T /*unused*/, D d) { | |
| #if HWY_TARGET != HWY_SCALAR | |
| DoTestMultiRotateRight(d); | |
| DoGe128BitMultiRotateRightTests(d); | |
| #else | |
| (void)d; | |
| #endif | |
| } | |
| }; | |
| HWY_NOINLINE void TestAllMultiRotateRight() { | |
| #if HWY_HAVE_INTEGER64 | |
| const ForGEVectors<64, TestMultiRotateRight> test64; | |
| test64(uint64_t()); | |
| test64(int64_t()); | |
| #endif | |
| } | |
| } // namespace | |
| // NOLINTNEXTLINE(google-readability-namespace-comments) | |
| } // namespace HWY_NAMESPACE | |
| } // namespace hwy | |
| HWY_AFTER_NAMESPACE(); | |
| #if HWY_ONCE | |
| namespace hwy { | |
| namespace { | |
| HWY_BEFORE_TEST(HwyShiftTest); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllShifts); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllVariableShifts); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllRoundingShiftRight); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllShiftRightAndDemoteTo); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllVariableRoundingShr); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllMaskedShift); | |
| HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllMultiRotateRight); | |
| HWY_AFTER_TEST(); | |
| } // namespace | |
| } // namespace hwy | |
| HWY_TEST_MAIN(); | |
| #endif // HWY_ONCE |