Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package simdutf for openSUSE:Factory checked in at 2026-10-30 18:09:37 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/simdutf (Old) and /work/SRC/openSUSE:Factory/.simdutf.new.1255 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "simdutf" Fri Oct 30 18:09:37 2026 rev:11 rq:1393564 version:9.3.3 Changes: -------- --- /work/SRC/openSUSE:Factory/simdutf/simdutf.changes 2026-10-23 20:21:31.553569739 +0200 +++ /work/SRC/openSUSE:Factory/.simdutf.new.1255/simdutf.changes 2026-10-30 18:13:26.297125606 +0100 @@ -1,0 +2,9 @@ +Wed Oct 28 08:40:07 UTC 2026 - Dominique Leuenberger <[email protected]> + +- Update to version 9.3.3: + * Check the tail of the ASCII validators in place + * Check the tail of the UTF-8 validators in place + * base64: avoid overflow in base64_length_from_binary_with_lines + * arm64: fix base64 default_or_url tables accepting '[' + +------------------------------------------------------------------- Old: ---- simdutf-9.3.2.tar.xz New: ---- simdutf-9.3.3.tar.xz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ simdutf.spec ++++++ --- /var/tmp/diff_new_pack.3X6SAK/_old 2026-10-30 18:13:29.649265934 +0100 +++ /var/tmp/diff_new_pack.3X6SAK/_new 2026-10-30 18:13:29.651266018 +0100 @@ -19,7 +19,7 @@ %define lib_ver 37.0.0 %define so_ver 37 Name: simdutf -Version: 9.3.2 +Version: 9.3.3 Release: 0 Summary: Unicode validation and transcoding at billions of characters per second ++++++ _scmsync.obsinfo ++++++ --- /var/tmp/diff_new_pack.3X6SAK/_old 2026-10-30 18:13:29.748270078 +0100 +++ /var/tmp/diff_new_pack.3X6SAK/_new 2026-10-30 18:13:29.760270581 +0100 @@ -1,7 +1,7 @@ -mtime: 1792237928 -commit: a1e11eb2e43ced22e8258ca57f30f4e436eb2979470c3daab35464bb94f9ae2e +mtime: 1793177041 +commit: df5d5e885b4ddfb74f4063a37fd59d40d238d76577e5ee16f2def29d6b606a08 url: https://src.opensuse.org/GNOME/simdutf -revision: a1e11eb2e43ced22e8258ca57f30f4e436eb2979470c3daab35464bb94f9ae2e +revision: df5d5e885b4ddfb74f4063a37fd59d40d238d76577e5ee16f2def29d6b606a08 trackingbranch: factory projectscmsync: https://src.opensuse.org/GNOME/_ObsPrj ++++++ _service ++++++ --- /var/tmp/diff_new_pack.3X6SAK/_old 2026-10-30 18:13:29.820273093 +0100 +++ /var/tmp/diff_new_pack.3X6SAK/_new 2026-10-30 18:13:29.833273637 +0100 @@ -3,7 +3,7 @@ <service name="obs_scm" mode="manual"> <param name="scm">git</param> <param name="url">https://github.com/simdutf/simdutf.git</param> - <param name="revision">v9.3.2</param> + <param name="revision">v9.3.3</param> <param name="versionformat">@PARENT_TAG@+@TAG_OFFSET@</param> <param name="versionrewrite-pattern">v?(.*)\+0</param> <param name="versionrewrite-replacement">\1</param> ++++++ build.specials.obscpio ++++++ ++++++ build.specials.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/.gitignore new/.gitignore --- old/.gitignore 1970-01-01 01:00:00.000000000 +0100 +++ new/.gitignore 2026-10-28 09:44:01.000000000 +0100 @@ -0,0 +1,5 @@ +*.obscpio +*.osc +_build.* +.pbuild +osc-collab.* ++++++ simdutf-9.3.2.tar.xz -> simdutf-9.3.3.tar.xz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/CMakeLists.txt new/simdutf-9.3.3/CMakeLists.txt --- old/simdutf-9.3.2/CMakeLists.txt 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/CMakeLists.txt 2026-10-28 00:10:33.000000000 +0100 @@ -3,7 +3,7 @@ project(simdutf DESCRIPTION "Fast Unicode validation, transcoding and processing" LANGUAGES C CXX - VERSION 9.3.2 + VERSION 9.3.3 ) include (TestBigEndian) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/Doxyfile new/simdutf-9.3.3/Doxyfile --- old/simdutf-9.3.2/Doxyfile 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/Doxyfile 2026-10-28 00:10:33.000000000 +0100 @@ -38,7 +38,7 @@ # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "9.3.2" +PROJECT_NUMBER = "9.3.3" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/README.md new/simdutf-9.3.3/README.md --- old/simdutf-9.3.2/README.md 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/README.md 2026-10-28 00:10:33.000000000 +0100 @@ -149,7 +149,7 @@ 1. Pull the library in a directory ``` - wget https://github.com/simdutf/simdutf/releases/download/v9.3.2/singleheader.zip + wget https://github.com/simdutf/simdutf/releases/download/v9.3.3/singleheader.zip unzip singleheader.zip ``` You can replace `wget` by `curl -OL https://...` if you prefer. @@ -190,7 +190,7 @@ ## Single-header version -You can create a single-header version of the library where all of the code is put into two files (`simdutf.h` and `simdutf.cpp`). We publish a zip archive containing these files, e.g., see https://github.com/simdutf/simdutf/releases/download/v9.3.2/singleheader.zip +You can create a single-header version of the library where all of the code is put into two files (`simdutf.h` and `simdutf.cpp`). We publish a zip archive containing these files, e.g., see https://github.com/simdutf/simdutf/releases/download/v9.3.3/singleheader.zip You may generate it on your own using a Python script. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/include/simdutf/scalar/base64.h new/simdutf-9.3.3/include/simdutf/scalar/base64.h --- old/simdutf-9.3.2/include/simdutf/scalar/base64.h 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/include/simdutf/scalar/base64.h 2026-10-28 00:10:33.000000000 +0100 @@ -863,8 +863,12 @@ if (line_length < 4) { line_length = 4; } - size_t lines = - (base64_length + line_length - 1) / line_length; // number of lines + // number of lines, rounded up without computing base64_length + line_length, + // which wraps around when line_length is close to SIZE_MAX + size_t lines = base64_length / line_length; + if (base64_length % line_length != 0) { + lines++; + } return base64_length + lines - 1; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/include/simdutf/simdutf_version.h new/simdutf-9.3.3/include/simdutf/simdutf_version.h --- old/simdutf-9.3.2/include/simdutf/simdutf_version.h 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/include/simdutf/simdutf_version.h 2026-10-28 00:10:33.000000000 +0100 @@ -4,7 +4,7 @@ #define SIMDUTF_SIMDUTF_VERSION_H /** The version of simdutf being used (major.minor.revision) */ -#define SIMDUTF_VERSION "9.3.2" +#define SIMDUTF_VERSION "9.3.3" namespace simdutf { enum { @@ -19,7 +19,7 @@ /** * The revision (major.minor.REVISION) of simdutf being used. */ - SIMDUTF_VERSION_REVISION = 2 + SIMDUTF_VERSION_REVISION = 3 }; } // namespace simdutf diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/src/arm64/arm_base64.cpp new/simdutf-9.3.3/src/arm64/arm_base64.cpp --- old/simdutf-9.3.2/src/arm64/arm_base64.cpp 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/src/arm64/arm_base64.cpp 2026-10-28 00:10:33.000000000 +0100 @@ -335,7 +335,7 @@ if (default_or_url) { lut_lo = simdutf_make_uint8x16_t(0xa9, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, - 0xf8, 0xf9, 0xf1, 0xa2, 0xa1, 0xa5, 0xa0, 0xa6); + 0xf8, 0xf9, 0xf1, 0xa2, 0xa1, 0xa5, 0xa0, 0xb6); } else if (base64_url) { lut_lo = simdutf_make_uint8x16_t(0xa9, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, @@ -348,7 +348,7 @@ #else if (default_or_url) { lut_lo = uint8x16_t{0xa9, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, - 0xf8, 0xf9, 0xf1, 0xa2, 0xa1, 0xa5, 0xa0, 0xa6}; + 0xf8, 0xf9, 0xf1, 0xa2, 0xa1, 0xa5, 0xa0, 0xb6}; } else if (base64_url) { lut_lo = uint8x16_t{0xa9, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf9, 0xf1, 0xa0, 0xa1, 0xa5, 0xa0, 0xa2}; @@ -366,10 +366,10 @@ if (default_or_url) { lut_hi = simdutf_make_uint8x16_t(0x0, 0x1, 0x0, 0x0, 0x1, 0x6, 0x8, 0x8, 0x10, - 0x20, 0x20, 0x12, 0x40, 0x80, 0x80, 0x40); + 0x20, 0x20, 0x10, 0x40, 0x80, 0x80, 0x40); } else if (base64_url) { lut_hi = - simdutf_make_uint8x16_t(0x0, 0x1, 0x0, 0x0, 0x1, 0x6, 0x8, 0x8, 0x10, + simdutf_make_uint8x16_t(0x0, 0x1, 0x0, 0x0, 0x1, 0x4, 0x8, 0x8, 0x10, 0x20, 0x20, 0x12, 0x40, 0x80, 0x80, 0x40); } else { lut_hi = @@ -379,7 +379,7 @@ #else if (default_or_url) { lut_hi = uint8x16_t{0x0, 0x1, 0x0, 0x0, 0x1, 0x6, 0x8, 0x8, - 0x10, 0x20, 0x20, 0x12, 0x40, 0x80, 0x80, 0x40}; + 0x10, 0x20, 0x20, 0x10, 0x40, 0x80, 0x80, 0x40}; } else if (base64_url) { lut_hi = uint8x16_t{0x0, 0x1, 0x0, 0x0, 0x1, 0x4, 0x8, 0x8, 0x10, 0x20, 0x20, 0x12, 0x40, 0x80, 0x80, 0x40}; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/src/generic/ascii_validation.h new/simdutf-9.3.3/src/generic/ascii_validation.h --- old/simdutf-9.3.2/src/generic/ascii_validation.h 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/src/generic/ascii_validation.h 2026-10-28 00:10:33.000000000 +0100 @@ -18,17 +18,14 @@ count += 64; } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64<uint8_t> in(block); - if (!in.is_ascii()) { - SIMDUTF_VZEROUPPER(); - result res = scalar::ascii::validate_with_errors( - reinterpret_cast<const char *>(input + count), length - count); - return result(res.error, count + res.count); - } else { + if (remainder_is_ascii(reinterpret_cast<const uint8_t *>(input), length, + reader.block_index())) { return result(error_code::SUCCESS, length); } + SIMDUTF_VZEROUPPER(); + result res = scalar::ascii::validate_with_errors( + reinterpret_cast<const char *>(input + count), length - count); + return result(res.error, count + res.count); } bool generic_validate_ascii(const char *input, size_t length) { @@ -40,10 +37,8 @@ } reader.advance(); } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64<uint8_t> in(block); - return in.is_ascii(); + return remainder_is_ascii(reinterpret_cast<const uint8_t *>(input), length, + reader.block_index()); } } // namespace ascii_validation diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/src/generic/buf_block_reader.h new/simdutf-9.3.3/src/generic/buf_block_reader.h --- old/simdutf-9.3.2/src/generic/buf_block_reader.h 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/src/generic/buf_block_reader.h 2026-10-28 00:10:33.000000000 +0100 @@ -2,6 +2,51 @@ namespace SIMDUTF_IMPLEMENTATION { namespace { +template <typename T> simdutf_really_inline T load_bytes_as(const uint8_t *p) { + T v; + std::memcpy(&v, p, sizeof(T)); + return v; +} + +// Returns true if the n bytes at p are all ASCII, reading only those bytes. +// The validators use it on the last, partial block instead of copying that +// block to the stack, which would stall the vector load of the copy. +simdutf_really_inline bool tail_is_ascii(const uint8_t *p, size_t n) { + uint64_t acc = 0; + if (n >= 16) { + for (size_t i = 0; i + 16 < n; i += 16) { + acc |= + load_bytes_as<uint64_t>(p + i) | load_bytes_as<uint64_t>(p + i + 8); + } + acc |= load_bytes_as<uint64_t>(p + n - 16) | + load_bytes_as<uint64_t>(p + n - 8); + } else if (n >= 8) { + acc = load_bytes_as<uint64_t>(p) | load_bytes_as<uint64_t>(p + n - 8); + } else if (n >= 4) { + acc = load_bytes_as<uint32_t>(p) | load_bytes_as<uint32_t>(p + n - 4); + } else { + for (size_t i = 0; i < n; i++) { + acc |= p[i]; + } + } + return (acc & 0x8080808080808080) == 0; +} + +// Like tail_is_ascii(buf + idx, len - idx), for the remainder that +// buf_block_reader leaves at idx (1 to 64 bytes, unless len is 0). The caller +// must already know that the bytes before idx are ASCII. Inputs of at least +// one block are checked with one load of their last block instead, which +// overlaps bytes that were already checked: a non-ASCII byte there can only be +// at or after idx. +simdutf_really_inline bool remainder_is_ascii(const uint8_t *buf, size_t len, + size_t idx) { + constexpr size_t block = sizeof(simd::simd8x64<uint8_t>); + if (len >= block) { + return simd::simd8x64<uint8_t>(buf + len - block).is_ascii(); + } + return tail_is_ascii(buf + idx, len - idx); +} + // Walks through a buffer in block-sized increments, loading the last part with // spaces template <size_t STEP_SIZE> struct buf_block_reader { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/src/generic/utf8_validation/utf8_lookup4_algorithm.h new/simdutf-9.3.3/src/generic/utf8_validation/utf8_lookup4_algorithm.h --- old/simdutf-9.3.2/src/generic/utf8_validation/utf8_lookup4_algorithm.h 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/src/generic/utf8_validation/utf8_lookup4_algorithm.h 2026-10-28 00:10:33.000000000 +0100 @@ -238,6 +238,16 @@ } } + // Checks a block that may overlap bytes already checked, given the register + // of input just before it. Each byte is checked against the same preceding + // bytes as before, so the overlapping bytes give the same result. + simdutf_really_inline void + check_overlapping_input(const simd8x64<uint8_t> &input, + const simd8<uint8_t> prev_input) { + this->prev_input_block = prev_input; + this->check_next_input(input); + } + // do not forget to call check_eof! simdutf_really_inline bool errors() const { return this->error.any_bits_set_anywhere(); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/src/generic/utf8_validation/utf8_validator.h new/simdutf-9.3.3/src/generic/utf8_validation/utf8_validator.h --- old/simdutf-9.3.2/src/generic/utf8_validation/utf8_validator.h 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/src/generic/utf8_validation/utf8_validator.h 2026-10-28 00:10:33.000000000 +0100 @@ -3,6 +3,42 @@ namespace { namespace utf8_validation { +// Checks the bytes from idx on, after the last full block. When the bytes +// before the window (the last 64 bytes of the input) fill a register, the +// window is checked in place: the bytes that the last full block already +// checked are checked again with the same preceding bytes, so they give the +// same result. An ASCII tail cannot start or finish a multibyte character, so +// it only needs check_eof(), which reports a character that the last full +// block left incomplete. Shorter inputs copy the tail to the stack. +template <class checker> +simdutf_really_inline void check_remainder(checker &c, const uint8_t *input, + size_t length, size_t idx) { + constexpr size_t block = sizeof(simd::simd8x64<uint8_t>); + constexpr size_t width = sizeof(simd::simd8<uint8_t>); + static_assert(block == 8 * sizeof(uint64_t), "one mask bit per byte"); + if (length >= block + width) { + // The window starts at or before idx: buf_block_reader leaves 1 to 64 + // bytes, so 0 <= idx - start < 64. + const size_t start = length - block; + const simd::simd8x64<uint8_t> in(input + start); + // Bit i is set if byte start + i is not ASCII, so the bytes from idx on + // are the top length - idx bits. is_ascii() goes first because GCC, up + // to at least GCC 17, builds gteq_unsigned(0x80) from min and compare + // instead of one movemask, which costs ASCII tails about 0.5 ns. + if (!in.is_ascii() && (in.gteq_unsigned(0x80) >> (idx - start)) != 0) { + c.check_overlapping_input(in, + simd::simd8<uint8_t>(input + start - width)); + } + } else if (idx != length) { + // Only an empty input leaves no bytes. Its pointer may be null, and + // memcpy from a null pointer is undefined. + uint8_t buffer[block]{}; + std::memcpy(buffer, input + idx, length - idx); + c.check_next_input(simd::simd8x64<uint8_t>(buffer)); + } + c.check_eof(); +} + /** * Validates that the string is actual UTF-8. */ @@ -15,12 +51,7 @@ c.check_next_input(in); reader.advance(); } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64<uint8_t> in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); + check_remainder(c, input, length, reader.block_index()); return !c.errors(); } @@ -54,12 +85,7 @@ reader.advance(); count += 64; } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64<uint8_t> in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); + check_remainder(c, input, length, reader.block_index()); if (c.errors()) { if (count != 0) { count--; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/tests/base64_tests.cpp new/simdutf-9.3.3/tests/base64_tests.cpp --- old/simdutf-9.3.2/tests/base64_tests.cpp 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/tests/base64_tests.cpp 2026-10-28 00:10:33.000000000 +0100 @@ -6,6 +6,7 @@ #include <cstddef> #include <cstring> #include <iostream> +#include <limits> #include <string> #include <tuple> #include <vector> @@ -339,6 +340,64 @@ text); } +TEST(base64_every_byte_value_in_a_block) { + const simdutf::base64_options options[] = { + simdutf::base64_default, + simdutf::base64_url, + simdutf::base64_default_or_url, + simdutf::base64_default_accept_garbage, + simdutf::base64_url_accept_garbage, + simdutf::base64_default_or_url_accept_garbage}; + for (simdutf::base64_options option : options) { + const bool url = option == simdutf::base64_url || + option == simdutf::base64_url_accept_garbage; + const bool both = option == simdutf::base64_default_or_url || + option == simdutf::base64_default_or_url_accept_garbage; + const bool garbage = + option == simdutf::base64_default_accept_garbage || + option == simdutf::base64_url_accept_garbage || + option == simdutf::base64_default_or_url_accept_garbage; + for (int c = 0; c < 256; c++) { + if (c == '=') { + continue; + } + const bool alpha = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9'); + const bool extra = ((c == '+' || c == '/') && (!url || both)) || + ((c == '-' || c == '_') && (url || both)); + const bool space = + c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; + for (size_t pos : {size_t(0), size_t(31), size_t(63)}) { + std::string input(128, 'A'); + input[pos] = char(c); + std::vector<char> out(96); + simdutf::result r = implementation.base64_to_binary( + input.data(), input.size(), out.data(), option); + if (alpha || extra) { + ASSERT_EQUAL(r.error, simdutf::error_code::SUCCESS); + ASSERT_EQUAL(r.count, 96); + } else if (space || garbage) { + ASSERT_EQUAL(r.error, simdutf::error_code::SUCCESS); + ASSERT_EQUAL(r.count, 95); + } else { + ASSERT_EQUAL(r.error, simdutf::error_code::INVALID_BASE64_CHARACTER); + ASSERT_EQUAL(r.count, pos); + } + } + } + } +} + +TEST(base64_default_or_url_only_garbage) { + const std::string input = "[[" + std::string(62, '!'); + std::vector<char> out(48); + simdutf::result r = implementation.base64_to_binary( + input.data(), input.size(), out.data(), + simdutf::base64_default_or_url_accept_garbage); + ASSERT_EQUAL(r.error, simdutf::error_code::SUCCESS); + ASSERT_EQUAL(r.count, 0); +} + TEST(issue_dash) { const std::string input = "Iw=="; std::vector<char> back(1); @@ -636,6 +695,24 @@ } } } + +// A line length close to SIZE_MAX used to make the line count wrap around, +// so base64_length_from_binary_with_lines returned one byte less than +// binary_to_base64_with_lines writes. +TEST(with_lines_huge_line_length) { + const size_t max_size = std::numeric_limits<size_t>::max(); + std::vector<char> source(200, 'f'); + for (size_t line_length : {max_size, max_size - 1, max_size - 100}) { + for (size_t len = 1; len < source.size(); len++) { + std::vector<char> buffer(simdutf::base64_length_from_binary_with_lines( + len, simdutf::base64_default, line_length)); + ASSERT_EQUAL(buffer.size(), simdutf::base64_length_from_binary(len)); + size_t size = implementation.binary_to_base64_with_lines( + source.data(), len, buffer.data(), line_length); + ASSERT_EQUAL(size, buffer.size()); + } + } +} // stop-before-partial should behave like so: // 1. if the last chunk is not a multiple of 4, we should stop before the last diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/tests/validate_ascii_basic_tests.cpp new/simdutf-9.3.3/tests/validate_ascii_basic_tests.cpp --- old/simdutf-9.3.2/tests/validate_ascii_basic_tests.cpp 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/tests/validate_ascii_basic_tests.cpp 2026-10-28 00:10:33.000000000 +0100 @@ -1,5 +1,11 @@ #include "simdutf.h" +#include <algorithm> +#include <cstdio> +#include <string> +#include <string_view> +#include <vector> + #include <tests/helpers/test.h> #include <tests/helpers/fixed_string.h> @@ -162,4 +168,56 @@ })); } #endif +namespace { +// The validators handle the bytes after the last full 64-byte block +// separately. These tests check every length of that tail against the scalar +// code. Each input is copied to a buffer of exactly its length, so that the +// address sanitizer reports a read past either end. +constexpr size_t tail_max_len = 200; + +void check_tail(const simdutf::implementation &implementation, + const std::string &input) { + const std::vector<char> buffer(input.begin(), input.end()); + const char *p = buffer.data(); + const size_t len = buffer.size(); + const bool valid = implementation.validate_ascii(p, len); + const simdutf::result with_errors = + implementation.validate_ascii_with_errors(p, len); + const simdutf::result expected = + simdutf::scalar::ascii::validate_with_errors(p, len); + if (valid != (expected.error == simdutf::error_code::SUCCESS) || + with_errors.error != expected.error || + with_errors.count != expected.count) { + std::printf("failing input, %zu bytes:", len); + for (size_t i = 0; i < len; i++) { + std::printf(" %02x", static_cast<unsigned char>(p[i])); + } + std::printf("\n"); + } + ASSERT_EQUAL(valid, simdutf::scalar::ascii::validate(p, len)); + ASSERT_EQUAL(with_errors.error, expected.error); + ASSERT_EQUAL(with_errors.count, expected.count); +} +} // namespace + +TEST(tail_ascii_every_length) { + for (size_t len = 0; len <= tail_max_len; len++) { + check_tail(implementation, std::string(len, 'a')); + } +} + +TEST(tail_non_ascii_at_every_position) { + constexpr std::string_view patterns[] = {"\x80", "\xff", "\xc3\xa9"}; + for (size_t len = 1; len <= tail_max_len; len++) { + for (std::string_view pattern : patterns) { + for (size_t pos = 0; pos < len; pos++) { + std::string s(len, 'a'); + s.replace(pos, std::min(pattern.size(), len - pos), pattern.data(), + std::min(pattern.size(), len - pos)); + check_tail(implementation, s); + } + } + } +} + TEST_MAIN diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/simdutf-9.3.2/tests/validate_utf8_basic_tests.cpp new/simdutf-9.3.3/tests/validate_utf8_basic_tests.cpp --- old/simdutf-9.3.2/tests/validate_utf8_basic_tests.cpp 2026-10-17 05:05:59.000000000 +0200 +++ new/simdutf-9.3.3/tests/validate_utf8_basic_tests.cpp 2026-10-28 00:10:33.000000000 +0100 @@ -1,6 +1,10 @@ #include "simdutf.h" +#include <algorithm> +#include <cstdio> +#include <string> #include <string_view> +#include <vector> #include <tests/helpers/fixed_string.h> #include <tests/helpers/test.h> @@ -173,4 +177,103 @@ #endif +namespace { +// The validators handle the bytes after the last full 64-byte block +// separately. These tests check every length of that tail against the scalar +// code. Each input is copied to a buffer of exactly its length, so that the +// address sanitizer reports a read past either end. +constexpr size_t tail_max_len = 200; + +// Bytes that end an ASCII prefix: invalid bytes, complete characters and +// characters cut off by the end of the input. +constexpr std::string_view tail_patterns[] = { + "\x80", "\xff", "\xc3\xa9", "\xc3", + "\xe4\xb8\xad", "\xe4\xb8", "\xed\xa0\x80", "\xf0\x9f\x98\x80", + "\xf0\x9f\x98"}; + +void check_tail(const simdutf::implementation &implementation, + const std::string &input) { + const std::vector<char> buffer(input.begin(), input.end()); + const char *p = buffer.data(); + const size_t len = buffer.size(); + const bool valid = implementation.validate_utf8(p, len); + const simdutf::result with_errors = + implementation.validate_utf8_with_errors(p, len); + const simdutf::result expected = + simdutf::scalar::utf8::validate_with_errors(p, len); + if (valid != (expected.error == simdutf::error_code::SUCCESS) || + with_errors.error != expected.error || + with_errors.count != expected.count) { + std::printf("failing input, %zu bytes:", len); + for (size_t i = 0; i < len; i++) { + std::printf(" %02x", static_cast<unsigned char>(p[i])); + } + std::printf("\n"); + } + ASSERT_EQUAL(valid, simdutf::scalar::utf8::validate(p, len)); + ASSERT_EQUAL(with_errors.error, expected.error); + ASSERT_EQUAL(with_errors.count, expected.count); +} + +// Repeats character to fill len bytes, cutting off the last one if needed. +std::string repeat(std::string_view character, size_t len) { + std::string s; + while (s.size() < len) { + s += character; + } + s.resize(len); + return s; +} +} // namespace + +TEST(tail_ascii_every_length) { + for (size_t len = 0; len <= tail_max_len; len++) { + check_tail(implementation, std::string(len, 'a')); + } +} + +TEST(tail_non_ascii_at_every_position) { + for (size_t len = 1; len <= tail_max_len; len++) { + for (std::string_view pattern : tail_patterns) { + for (size_t pos = 0; pos < len; pos++) { + std::string s(len, 'a'); + s.replace(pos, std::min(pattern.size(), len - pos), pattern.data(), + std::min(pattern.size(), len - pos)); + check_tail(implementation, s); + } + } + } +} + +TEST(tail_multibyte_text_with_ascii_tail) { + for (size_t len = 0; len <= tail_max_len; len++) { + for (size_t tail = 0; tail <= len && tail <= 70; tail++) { + const size_t text = (len - tail) / 3 * 3; + check_tail(implementation, + repeat("\xe4\xb8\xad", text) + std::string(len - text, 'b')); + } + } +} + +// Long inputs check their last 64 bytes in place, which overlaps the last full +// block. Multibyte text of each width makes the last full block end inside a +// character at every offset, and the patterns put an error or a cut-off +// character at every position. +TEST(tail_non_ascii_in_multibyte_text) { + constexpr std::string_view characters[] = {"\xc3\xa9", "\xe4\xb8\xad", + "\xf0\x9f\x98\x80"}; + for (size_t len = 64; len <= 160; len++) { + for (std::string_view character : characters) { + for (std::string_view pattern : tail_patterns) { + for (size_t pos = 0; pos < len; pos++) { + std::string s = repeat(character, len); + s.replace(pos, std::min(pattern.size(), len - pos), pattern.data(), + std::min(pattern.size(), len - pos)); + check_tail(implementation, s); + } + } + } + } +} + TEST_MAIN ++++++ simdutf.obsinfo ++++++ --- /var/tmp/diff_new_pack.3X6SAK/_old 2026-10-30 18:13:31.435340703 +0100 +++ /var/tmp/diff_new_pack.3X6SAK/_new 2026-10-30 18:13:31.450341331 +0100 @@ -1,5 +1,5 @@ name: simdutf -version: 9.3.2 -mtime: 1792206359 -commit: 8d82299606e675b50de8dc16fd118aae4b65bfde +version: 9.3.3 +mtime: 1793142633 +commit: 59589cd3df6823cbfdbd253cde3377e7c3e1fe9b
