[third-party] update simdutf This CL updates simdut to v7.3.3 to fix the issue with `stop-before-partial` last chunk handling that was reported here https://github.com/simdutf/simdutf/issues/821 single header version is from: https://github.com/simdutf/simdutf/releases/tag/v7.3.3 BYPASS_LARGE_CHANGE_WARNING: third-party update No-Presubmit: true Change-Id: I26e6003133d1b112a8acaa1adcac4b2fe458227b Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6765991 Reviewed-by: Nidhi Jaju <nidhijaju@chromium.org> Commit-Queue: Rezvan Mahdavi Hezaveh <rezvan@chromium.org> Cr-Commit-Position: refs/heads/main@{#1489635} NOKEYCHECK=True GitOrigin-RevId: 036e4472a44c717461e8c7b2e0aeda4b11688602
diff --git a/README.chromium b/README.chromium index 3c7e9e5..53e335f 100644 --- a/README.chromium +++ b/README.chromium
@@ -1,7 +1,7 @@ Name: simdutf unicode transcoder Short Name: simdutf URL: https://github.com/simdutf/simdutf -Version: 7.3.0 +Version: 7.3.3 Update Mechanism: Manual License: MIT License File: LICENSE
diff --git a/simdutf.cpp b/simdutf.cpp index 45b5069..991df2d 100644 --- a/simdutf.cpp +++ b/simdutf.cpp
@@ -1,4 +1,4 @@ -/* auto-generated on 2025-05-28 18:43:22 -0400. Do not edit! */ +/* auto-generated on 2025-07-13 10:46:57 -0400. Do not edit! */ /* begin file src/simdutf.cpp */ #include "simdutf.h" @@ -940,7 +940,7 @@ } else if (length >= 4 && byte[0] == 0x00 and byte[1] == 0x00 and byte[2] == 0xfe and byte[3] == 0xff) { return encoding_type::UTF32_BE; - } else if (length >= 4 && byte[0] == 0xef and byte[1] == 0xbb and + } else if (length >= 3 && byte[0] == 0xef and byte[1] == 0xbb and byte[2] == 0xbf) { return encoding_type::UTF8; } @@ -7714,7 +7714,7 @@ char16_t *output) const noexcept final; void to_well_formed_utf16le(const char16_t *input, size_t len, char16_t *output) const noexcept final; -#endif +#endif // SIMDUTF_FEATURE_UTF16 }; } // namespace ppc64 @@ -10157,9 +10157,9 @@ } #if SIMDUTF_NEED_TRAILING_ZEROES -// simdutf_really_inline int trailing_zeroes(uint64_t input_num) { -// return __builtin_ctzll(input_num); -// } +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); +} #endif } // unnamed namespace @@ -11525,9 +11525,9 @@ } #if SIMDUTF_NEED_TRAILING_ZEROES -// simdutf_really_inline int trailing_zeroes(uint64_t input_num) { -// return __builtin_ctzll(input_num); -// } +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); +} #endif } // unnamed namespace @@ -13731,6 +13731,13 @@ } if (idx != 4) { simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + // The idea here is that in loose mode, // if there is padding at all, it must be used // to form 4-wise chunk. However, in loose mode, @@ -30180,6 +30187,9 @@ } if ((bufferptr - buffer_start) != 0) { + // For efficiency reasons, we end up reproducing much of the code + // in base64_tail_decode_impl. Better engineering would be to + // refactor the code so that we can call it without a performance hit. size_t rem = (bufferptr - buffer_start); int idx = rem % 4; __mmask64 mask = ((__mmask64)1 << rem) - 1; @@ -30196,7 +30206,12 @@ 28, 29, 30, 24, 25, 26, 20, 21, 22, 16, 17, 18, 12, 13, 14, 8, 9, 10, 4, 5, 6, 0, 1, 2); const __m512i shuffled = _mm512_permutexvar_epi8(pack, merged); - + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } // The idea here is that in loose mode, // if there is padding at all, it must be used // to form 4-wise chunk. However, in loose mode, @@ -47306,6 +47321,39 @@ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 /* end file src/rvv/rvv_utf8_to.inl.cpp */ +#if SIMDUTF_FEATURE_BASE64 +/* begin file src/rvv/rvv_find.cpp */ +simdutf_really_inline const char *util_find(const char *start, const char *end, + char character) noexcept { + const char *src = start; + for (size_t len = end - start, vl; len > 0; len -= vl, src += vl) { + vl = __riscv_vsetvl_e8m8(len); + vuint8m8_t v = __riscv_vle8_v_u8m8((uint8_t *)src, vl); + long idx = + __riscv_vfirst_m_b1(__riscv_vmseq_vx_u8m8_b1(v, character, vl), vl); + if (idx >= 0) + return src + idx; + } + return end; +} + +simdutf_really_inline const char16_t *util_find(const char16_t *start, + const char16_t *end, + char16_t character) noexcept { + const char16_t *src = start; + for (size_t len = end - start, vl; len > 0; len -= vl, src += vl) { + vl = __riscv_vsetvl_e16m8(len); + vuint16m8_t v = __riscv_vle16_v_u16m8((uint16_t *)src, vl); + long idx = + __riscv_vfirst_m_b2(__riscv_vmseq_vx_u16m8_b2(v, character, vl), vl); + if (idx >= 0) + return src + idx; + } + return end; +} +/* end file src/rvv/rvv_find.cpp */ +#endif // SIMDUTF_FEATURE_BASE64 + #if SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused int implementation::detect_encodings(const char *input, @@ -47401,12 +47449,12 @@ const char *implementation::find(const char *start, const char *end, char character) const noexcept { - return std::find(start, end, character); + return util_find(start, end, character); } const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { - return std::find(start, end, character); + return util_find(start, end, character); } #endif // SIMDUTF_FEATURE_BASE64 @@ -57237,6 +57285,68 @@ return {SUCCESS, srclen, size_t(dst - dstinit)}; } /* end file src/lsx/lsx_base64.cpp */ +/* begin file src/lsx/lsx_find.cpp */ +simdutf_really_inline const char *util_find(const char *start, const char *end, + char character) noexcept { + if (start >= end) + return end; + + const int step = 16; + __m128i char_vec = __lsx_vreplgr2vr_b(static_cast<uint8_t>(character)); + + while (end - start >= step) { + __m128i data = __lsx_vld(reinterpret_cast<const __m128i *>(start), 0); + __m128i cmp = __lsx_vseq_b(data, char_vec); + if (__lsx_bnz_v(cmp)) { + uint16_t mask = + static_cast<uint16_t>(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); + return start + trailing_zeroes(mask); + } + + start += step; + } + + // Handle remaining bytes with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + + return end; +} + +simdutf_really_inline const char16_t *util_find(const char16_t *start, + const char16_t *end, + char16_t character) noexcept { + if (start >= end) + return end; + + const int step = 8; + __m128i char_vec = __lsx_vreplgr2vr_h(static_cast<uint16_t>(character)); + + while (end - start >= step) { + __m128i data = __lsx_vld(reinterpret_cast<const __m128i *>(start), 0); + __m128i cmp = __lsx_vseq_h(data, char_vec); + if (__lsx_bnz_v(cmp)) { + uint16_t mask = + static_cast<uint16_t>(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); + return start + trailing_zeroes(mask) / 2; + } + + start += step; + } + + // Handle remaining elements with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + + return end; +} +/* end file src/lsx/lsx_find.cpp */ #endif // SIMDUTF_FEATURE_BASE64 } // namespace @@ -60748,12 +60858,12 @@ } const char *implementation::find(const char *start, const char *end, char character) const noexcept { - return std::find(start, end, character); + return util_find(start, end, character); } const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { - return std::find(start, end, character); + return util_find(start, end, character); } #endif // SIMDUTF_FEATURE_BASE64 @@ -64099,6 +64209,72 @@ return {SUCCESS, srclen, size_t(dst - dstinit)}; } /* end file src/lasx/lasx_base64.cpp */ +/* begin file src/lasx/lasx_find.cpp */ +simdutf_really_inline const char *util_find(const char *start, const char *end, + char character) noexcept { + if (start >= end) + return end; + + const int step = 32; + __m256i char_vec = __lasx_xvreplgr2vr_b(static_cast<uint16_t>(character)); + + while (end - start >= step) { + __m256i data = __lasx_xvld(reinterpret_cast<const __m256i *>(start), 0); + __m256i cmp = __lasx_xvseq_b(data, char_vec); + if (__lasx_xbnz_v(cmp)) { + __m256i res = __lasx_xvmsknz_b(cmp); + uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); + uint32_t mask = (mask0 | (mask1 << 16)); + return start + trailing_zeroes(mask); + } + + start += step; + } + + // Handle remaining bytes with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + + return end; +} + +simdutf_really_inline const char16_t *util_find(const char16_t *start, + const char16_t *end, + char16_t character) noexcept { + if (start >= end) + return end; + + const int step = 16; + __m256i char_vec = __lasx_xvreplgr2vr_h(static_cast<uint16_t>(character)); + + while (end - start >= step) { + __m256i data = __lasx_xvld(reinterpret_cast<const __m256i *>(start), 0); + __m256i cmp = __lasx_xvseq_h(data, char_vec); + if (__lasx_xbnz_v(cmp)) { + __m256i res = __lasx_xvmsknz_b(cmp); + uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); + uint32_t mask = (mask0 | (mask1 << 16)); + return start + trailing_zeroes(mask) / 2; + } + + start += step; + } + + // Handle remaining elements with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + + return end; +} +/* end file src/lasx/lasx_find.cpp */ #endif // SIMDUTF_FEATURE_BASE64 } // namespace @@ -67722,12 +67898,12 @@ const char *implementation::find(const char *start, const char *end, char character) const noexcept { - return std::find(start, end, character); + return util_find(start, end, character); } const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { - return std::find(start, end, character); + return util_find(start, end, character); } #endif // SIMDUTF_FEATURE_BASE64
diff --git a/simdutf.h b/simdutf.h index 8068a09..61e16d4 100644 --- a/simdutf.h +++ b/simdutf.h
@@ -1,4 +1,4 @@ -/* auto-generated on 2025-05-28 18:43:22 -0400. Do not edit! */ +/* auto-generated on 2025-07-13 10:46:57 -0400. Do not edit! */ /* begin file include/simdutf.h */ #ifndef SIMDUTF_H #define SIMDUTF_H @@ -628,7 +628,7 @@ OTHER // Not related to validation/transcoding. }; #if SIMDUTF_CPLUSPLUS17 -inline std::string_view error_to_string(error_code code) { +inline std::string_view error_to_string(error_code code) noexcept { switch (code) { case SUCCESS: return "SUCCESS"; @@ -664,16 +664,17 @@ // case of success, indicates the number of code units // validated/written. - simdutf_really_inline result() : error{error_code::SUCCESS}, count{0} {} + simdutf_really_inline result() noexcept + : error{error_code::SUCCESS}, count{0} {} - simdutf_really_inline result(error_code err, size_t pos) + simdutf_really_inline result(error_code err, size_t pos) noexcept : error{err}, count{pos} {} - simdutf_really_inline bool is_ok() const { + simdutf_really_inline bool is_ok() const noexcept { return error == error_code::SUCCESS; } - simdutf_really_inline bool is_err() const { + simdutf_really_inline bool is_err() const noexcept { return error != error_code::SUCCESS; } }; @@ -685,14 +686,14 @@ bool padding_error = false; // true if the error is due to padding, only // meaningful when error is not SUCCESS - simdutf_really_inline full_result() + simdutf_really_inline full_result() noexcept : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} simdutf_really_inline full_result(error_code err, size_t pos_in, - size_t pos_out) + size_t pos_out) noexcept : error{err}, input_count{pos_in}, output_count{pos_out} {} simdutf_really_inline full_result(error_code err, size_t pos_in, - size_t pos_out, bool padding_err) + size_t pos_out, bool padding_err) noexcept : error{err}, input_count{pos_in}, output_count{pos_out}, padding_error{padding_err} {} @@ -720,7 +721,7 @@ #define SIMDUTF_SIMDUTF_VERSION_H /** The version of simdutf being used (major.minor.revision) */ -#define SIMDUTF_VERSION "7.3.0" +#define SIMDUTF_VERSION "7.3.3" namespace simdutf { enum { @@ -735,7 +736,7 @@ /** * The revision (major.minor.REVISION) of simdutf being used. */ - SIMDUTF_VERSION_REVISION = 0 + SIMDUTF_VERSION_REVISION = 3 }; } // namespace simdutf @@ -1577,7 +1578,7 @@ #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 /** - * Convert Latin1 string into UTF8 string. + * Convert Latin1 string into UTF-8 string. * * This function is suitable to work with inputs from untrusted sources. * @@ -1600,7 +1601,7 @@ #endif // SIMDUTF_SPAN /** - * Convert Latin1 string into UTF8 string with output limit. + * Convert Latin1 string into UTF-8 string with output limit. * * This function is suitable to work with inputs from untrusted sources. * @@ -1638,7 +1639,7 @@ * * This function is suitable to work with inputs from untrusted sources. * - * @param input the Latin1 string to convert + * @param input the Latin1 string to convert * @param length the length of the string in bytes * @param utf16_buffer the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if conversion is not possible @@ -1777,7 +1778,7 @@ /** * Using native endianness, convert a Latin1 string into a UTF-16 string. * - * @param input the UTF-8 string to convert + * @param input the Latin1 string to convert * @param length the length of the string in bytes * @param utf16_buffer the pointer to buffer that can hold conversion result * @return the number of written char16_t. @@ -4039,7 +4040,7 @@ * maximal_binary_length_from_base64(input, length) bytes long. If you fail to * provide that much space, the function may cause a buffer overflow. * - * Advanced users may want to taylor how the last chunk is handled. By default, + * Advanced users may want to tailor how the last chunk is handled. By default, * we use a loose (forgiving) approach but we also support a strict approach * as well as a stop_before_partial approach, as per the following proposal: * @@ -4210,7 +4211,7 @@ * maximal_binary_length_from_base64(input, length) bytes long. If you fail * to provide that much space, the function may cause a buffer overflow. * - * Advanced users may want to taylor how the last chunk is handled. By default, + * Advanced users may want to tailor how the last chunk is handled. By default, * we use a loose (forgiving) approach but we also support a strict approach * as well as a stop_before_partial approach, as per the following proposal: * @@ -4337,7 +4338,7 @@ * true. In that case, the function will decode up to the first invalid * character. Extra padding characters ('=') are considered invalid characters. * - * Advanced users may want to taylor how the last chunk is handled. By default, + * Advanced users may want to tailor how the last chunk is handled. By default, * we use a loose (forgiving) approach but we also support a strict approach * as well as a stop_before_partial approach, as per the following proposal: * @@ -4804,7 +4805,7 @@ #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 /** - * Convert Latin1 string into UTF8 string. + * Convert Latin1 string into UTF-8 string. * * This function is suitable to work with inputs from untrusted sources. *