Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-mutf8 for openSUSE:Factory checked in at 2026-07-14 13:49:27 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-mutf8 (Old) and /work/SRC/openSUSE:Factory/.python-mutf8.new.1991 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-mutf8" Tue Jul 14 13:49:27 2026 rev:3 rq:1365421 version:1.1.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-mutf8/python-mutf8.changes 2025-06-05 20:36:39.675086319 +0200 +++ /work/SRC/openSUSE:Factory/.python-mutf8.new.1991/python-mutf8.changes 2026-07-14 13:50:39.728491670 +0200 @@ -1,0 +2,14 @@ +Mon Jul 13 16:33:44 UTC 2026 - Dirk Müller <[email protected]> + +- update to 1.1.0: + * Supports python 3.9 to 3.14. + * Fixed 3 issues with surrogate pair decoding & encoding + (closes #6, #4, #5). + * *Significantly* improved performance on recent Python + versions when decoding MUTF-8 strings that are really UTF-8 + (the typical case 90% of the time), improving decodes of 1kb + strings by ~5x and 64kb strings by ~9x. Encoding of 1kb + strings is ~12x faster and 64kb strings are ~48x faster. + * Reduced encoding allocations for less memory churn. + +------------------------------------------------------------------- Old: ---- mutf8-1.0.6.tar.gz New: ---- mutf8-1.1.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-mutf8.spec ++++++ --- /var/tmp/diff_new_pack.HNb0g5/_old 2026-07-14 13:50:40.204508056 +0200 +++ /var/tmp/diff_new_pack.HNb0g5/_new 2026-07-14 13:50:40.204508056 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-mutf8 # -# Copyright (c) 2025 SUSE LLC +# Copyright (c) 2026 SUSE LLC and contributors # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,7 +17,7 @@ Name: python-mutf8 -Version: 1.0.6 +Version: 1.1.0 Release: 0 Summary: Python/C encoders/decoders for MUTF-8/CESU-8 License: MIT ++++++ mutf8-1.0.6.tar.gz -> mutf8-1.1.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/.bumpversion.cfg new/mutf8-1.1.0/.bumpversion.cfg --- old/mutf8-1.0.6/.bumpversion.cfg 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/.bumpversion.cfg 1970-01-01 01:00:00.000000000 +0100 @@ -1,6 +0,0 @@ -[bumpversion] -current_version = 1.0.6 -commit = True -tag = True - -[bumpversion:file:setup.py] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/.github/workflows/release.yml new/mutf8-1.1.0/.github/workflows/release.yml --- old/mutf8-1.0.6/.github/workflows/release.yml 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/.github/workflows/release.yml 2026-07-10 20:04:18.000000000 +0200 @@ -1,99 +1,71 @@ +name: CI + on: push: - # Sequence of patterns matched against refs/tags - tags: - - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 - -name: Release + branches: [master] + pull_request: + release: + types: [published] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build: - name: Creating Windows and OSX wheels - + build_wheels: + name: Wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install wheel twine - pip install -e '.' - - - name: Publishing - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py bdist_wheel - twine upload dist/* + - name: Build and test wheels + uses: pypa/[email protected] - build_manylinux2010: - name: Creating manylinux releases + - uses: actions/upload-artifact@v7 + with: + name: wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl + build_sdist: + name: Source distribution runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Build (manylinux2014) - uses: RalfG/[email protected]_x86_64 - with: - python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' - - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install wheel twine - - - name: Publishing - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload dist/*-manylinux*.whl - - sdist: - name: Creating source release + - uses: actions/checkout@v7 + - name: Install uv + uses: astral-sh/[email protected] + with: + enable-cache: true + + - name: Build sdist + run: uv build --sdist + + - uses: actions/upload-artifact@v7 + with: + name: sdist + path: ./dist/*.tar.gz + + publish: + name: Publish to PyPI + needs: [build_wheels, build_sdist] runs-on: ubuntu-latest - + if: github.event_name == 'release' && github.repository == 'TkTech/mutf8' + environment: + name: pypi + url: https://pypi.org/p/mutf8 + permissions: + id-token: write steps: - - uses: actions/checkout@v2 + - uses: actions/download-artifact@v8 + with: + path: dist + merge-multiple: true - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install wheel twine - - - name: Publishing - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist - twine upload dist/* + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/.github/workflows/test.yml new/mutf8-1.1.0/.github/workflows/test.yml --- old/mutf8-1.0.6/.github/workflows/test.yml 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/.github/workflows/test.yml 1970-01-01 01:00:00.000000000 +0100 @@ -1,39 +0,0 @@ -name: Tests - -on: [push] - -jobs: - build: - name: Running tests. - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - - steps: - # Python needs to be setup before checkout to prevent files from being - # left in the source tree. See setup-python/issues/106. - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/checkout@v2 - - - name: Installing platform dependencies - if: matrix.os == 'ubuntu-latest' - run: > - sudo apt-get install -y - build-essential - clang-6.0 - - - name: Installing python dependencies - run: | - python -m pip install --upgrade pip - pip install -e '.[test]' - - - name: Running tests - run: | - pytest diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/.gitignore new/mutf8-1.1.0/.gitignore --- old/mutf8-1.0.6/.gitignore 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/.gitignore 2026-07-10 20:04:18.000000000 +0200 @@ -1,9 +1,14 @@ *.pyc *.pyd venv +.venv/ .eggs cython_debug/* build/ _build/ +dist/ +wheelhouse/ *.egg-info *.so +uv.lock +*.idea diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/README.md new/mutf8-1.1.0/README.md --- old/mutf8-1.0.6/README.md 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/README.md 2026-07-10 20:04:18.000000000 +0200 @@ -1,10 +1,9 @@ - + # mutf-8 This package contains simple pure-python as well as C encoders and decoders for -the MUTF-8 character encoding. In most cases, you can also parse the even-rarer -CESU-8. +the MUTF-8 character encoding. These days, you'll most likely encounter MUTF-8 when working on files or protocols related to the JVM. Strings in a Java `.class` file are encoded using @@ -14,26 +13,15 @@ This library was extracted from [Lawu][], a Python library for working with JVM class files. -## 🎉 Installation +## Installation +Prebuilt binary packages are available for most platforms and Python versions. Install the package from PyPi: ``` pip install mutf8 ``` -Binary wheels are available for the following: - -| | py3.6 | py3.7 | py3.8 | py3.9 | -| ---------------- | ----- | ----- | ----- | ----- | -| OS X (x86_64) | y | y | y | y | -| Windows (x86_64) | y | y | y | y | -| Linux (x86_64) | y | y | y | y | - -If binary wheels are not available, it will attempt to build the C extension -from source with any C99 compiler. If it could not build, it will fall back -to a pure-python version. - ## Usage Encoding and decoding is simple: @@ -48,23 +36,48 @@ This module *does not* register itself globally as a codec, since importing should be side-effect-free. -## 📈 Benchmarks +## Benchmarks -The C extension is significantly faster - often 20x to 40x faster. +The C extension is dramatically faster than the pure-Python fallback — from +~10× on tiny strings (where per-call overhead dominates) to several hundred +times on realistic payloads, and over 1000× when encoding large ASCII. <!-- BENCHMARK START --> ### MUTF-8 Decoding -| Name | Min (μs) | Max (μs) | StdDev | Ops | -|------------------------------|------------|------------|----------|---------------| -| cmutf8-decode_modified_utf8 | 0.00009 | 0.00080 | 0.00000 | 9957678.56358 | -| pymutf8-decode_modified_utf8 | 0.00190 | 0.06040 | 0.00000 | 450455.96019 | + +| Input | C | Python | Speedup | +|-------|---|--------|--------:| +| ASCII (16 B) | 81.5 ± 26 ns | 1.04 ± 0.18 µs | 13× | +| ASCII (1 KB) | 185 ± 38 ns | 65.7 ± 8.5 µs | 356× | +| ASCII (64 KB) | 5.58 ± 0.66 µs | 4.35 ± 0.23 ms | 780× | +| CJK (16 B) | 127 ± 15 ns | 1.33 ± 0.15 µs | 10× | +| CJK (1 KB) | 681 ± 117 ns | 99.3 ± 9.4 µs | 146× | +| CJK (64 KB) | 31.8 ± 3.3 µs | 6.28 ± 0.063 ms | 198× | +| Emoji (16 B) | 100 ± 21 ns | 882 ± 122 ns | 9× | +| Emoji (1 KB) | 538 ± 119 ns | 79.6 ± 6.9 µs | 148× | +| Emoji (64 KB) | 26.2 ± 4.1 µs | 5.32 ± 0.13 ms | 203× | +| Mixed (16 B) | 118 ± 27 ns | 2.17 ± 0.16 µs | 18× | +| Mixed (1 KB) | 860 ± 162 ns | 76.7 ± 9.2 µs | 89× | +| Mixed (64 KB) | 38 ± 5 µs | 5.3 ± 0.48 ms | 139× | ### MUTF-8 Encoding -| Name | Min (μs) | Max (μs) | StdDev | Ops | -|------------------------------|------------|------------|----------|----------------| -| cmutf8-encode_modified_utf8 | 0.00008 | 0.00151 | 0.00000 | 11897361.05101 | -| pymutf8-encode_modified_utf8 | 0.00180 | 0.16650 | 0.00000 | 474390.98091 | + +| Input | C | Python | Speedup | +|-------|---|--------|--------:| +| ASCII (16 B) | 70.5 ± 5.8 ns | 1.23 ± 0.15 µs | 18× | +| ASCII (1 KB) | 109 ± 16 ns | 49 ± 6.2 µs | 449× | +| ASCII (64 KB) | 1.55 ± 0.27 µs | 2.93 ± 0.19 ms | 1889× | +| CJK (16 B) | 81.4 ± 12 ns | 2.02 ± 0.26 µs | 25× | +| CJK (1 KB) | 624 ± 127 ns | 108 ± 16 µs | 173× | +| CJK (64 KB) | 32.5 ± 2.9 µs | 6.8 ± 0.24 ms | 209× | +| Emoji (16 B) | 73 ± 10 ns | 1.17 ± 0.12 µs | 16× | +| Emoji (1 KB) | 621 ± 76 ns | 67 ± 8.5 µs | 108× | +| Emoji (64 KB) | 33.3 ± 6 µs | 3.98 ± 0.12 ms | 119× | +| Mixed (16 B) | 108 ± 97 ns | 2.48 ± 0.35 µs | 23× | +| Mixed (1 KB) | 1.39 ± 0.29 µs | 63.3 ± 3.5 µs | 45× | +| Mixed (64 KB) | 81.2 ± 14 µs | 3.91 ± 0.093 ms | 48× | + <!-- BENCHMARK END --> ## C Extension diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/mutf8/__init__.py new/mutf8-1.1.0/mutf8/__init__.py --- old/mutf8-1.0.6/mutf8/__init__.py 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/mutf8/__init__.py 2026-07-10 20:04:18.000000000 +0200 @@ -10,6 +10,8 @@ for MUTF-8/CESU-8 into the python core. """ +__version__ = "1.1.0" + try: from mutf8.cmutf8 import decode_modified_utf8, encode_modified_utf8 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/mutf8/cmutf8.c new/mutf8-1.1.0/mutf8/cmutf8.c --- old/mutf8-1.0.6/mutf8/cmutf8.c 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/mutf8/cmutf8.c 2026-07-10 20:04:18.000000000 +0200 @@ -1,6 +1,7 @@ #define PY_SSIZE_T_CLEAN #include <Python.h> #include <stdint.h> +#include <string.h> PyDoc_STRVAR(decode_doc, "Decodes a bytestring containing MUTF-8 as defined in section\n" @@ -32,6 +33,22 @@ // MUTF-8 input. uint8_t *buf = (uint8_t *)view.buf; + + // Fast path: without 0xED (leads every surrogate/6-byte form), 0xC0 (the + // NUL escape) or a raw 0x00 (illegal), the input is plain UTF-8. Offload + // to CPython's optimized decoder; on any error fall back to the strict + // loop below so the exception is still reported as mutf-8. + if (memchr(buf, 0xED, view.len) == NULL && + memchr(buf, 0xC0, view.len) == NULL && + memchr(buf, 0x00, view.len) == NULL) { + PyObject *fast = PyUnicode_DecodeUTF8((const char *)buf, view.len, NULL); + if (fast != NULL) { + PyBuffer_Release(&view); + return fast; + } + PyErr_Clear(); + } + // Array of temporary UCS-4 codepoints. // There's no point using PyUnicode_new and _WriteChar, because // it requires us to have iterated the string to get the maximum unicode @@ -74,33 +91,30 @@ uint8_t b2 = buf[ix + 1]; uint8_t b3 = buf[ix + 2]; - if (x == 0xED && (b2 & 0xF0) == 0xA0) { - if (ix + 5 >= view.len) { - return_err( - "6-byte codepoint started, but input too short" - " to finish."); - } - - // Possible six-byte codepoint. - uint8_t b4 = buf[ix + 3]; + if (x == 0xED && (b2 & 0xF0) == 0xA0 && ix + 5 < view.len && + buf[ix + 3] == 0xED && (buf[ix + 4] & 0xF0) == 0xB0) { + // Six-byte codepoint: a supplementary character written as a + // UTF-16 surrogate pair, each half encoded as its own 3-byte + // sequence. We only take this branch once a low surrogate is + // confirmed to follow; otherwise `x b2 b3` is a lone + // (unpaired) surrogate and is decoded as a normal 3-byte + // codepoint below. uint8_t b5 = buf[ix + 4]; uint8_t b6 = buf[ix + 5]; - - if (b4 == 0xED && (b5 & 0xF0) == 0xB0) { - // Definite six-byte codepoint. - x = ( - 0x10000 | - (b2 & 0x0F) << 0x10 | - (b3 & 0x3F) << 0x0A | - (b5 & 0x0F) << 0x06 | - (b6 & 0x3F) - ); - ix += 5; - cp_out[cp_count++] = x; - continue; - } + x = ( + 0x10000 + + ((b2 & 0x0F) << 0x10 | + (b3 & 0x3F) << 0x0A | + (b5 & 0x0F) << 0x06 | + (b6 & 0x3F)) + ); + ix += 5; + cp_out[cp_count++] = x; + continue; } + // Regular three-byte codepoint. This also covers lone/unpaired + // surrogates, each of which is encoded as a single 3-byte sequence. x = ( (x & 0x0F) << 0x0C | (b2 & 0x3F) << 0x06 | @@ -183,11 +197,23 @@ void *data = PyUnicode_DATA(src); Py_ssize_t length = PyUnicode_GET_LENGTH(src); int kind = PyUnicode_KIND(src); - char *byte_out = PyMem_Calloc(_encoded_size(data, length, kind), 1); - if (!byte_out) { - return PyErr_NoMemory(); + // Fast path: with no supplementary char (kind < 4-byte) and no embedded + // NUL, MUTF-8 == UTF-8 with "surrogatepass" (same 3-byte form for lone + // surrogates). Offload to CPython's optimized encoder. + if (kind != PyUnicode_4BYTE_KIND && + PyUnicode_FindChar(src, 0, 0, length, 1) == -1) { + return PyUnicode_AsEncodedString(src, "utf-8", "surrogatepass"); + } + + // Allocate the result up front at its exact size and write straight into + // it, avoiding a temporary buffer, its zeroing, and a full-output copy. + Py_ssize_t size = _encoded_size(data, length, kind); + PyObject *out = PyBytes_FromStringAndSize(NULL, size); + if (out == NULL) { + return NULL; } + char *byte_out = PyBytes_AS_STRING(out); Py_ssize_t byte_count = 0; @@ -214,7 +240,11 @@ byte_out[byte_count++] = (0x80 | (0x3F & cp)); } else { - // "Two-times-three" byte codepoint. + // Six-byte codepoint: a supplementary character written as a + // UTF-16 surrogate pair, each half encoded as its own 3-byte + // sequence. The codepoint is first offset by 0x10000 as required + // by the surrogate-pair algorithm. + cp -= 0x10000; byte_out[byte_count++] = 0xED; byte_out[byte_count++] = 0xA0 | ((cp >> 0x10) & 0x0F); byte_out[byte_count++] = 0x80 | ((cp >> 0x0A) & 0x3F); @@ -224,8 +254,6 @@ } } - PyObject *out = PyBytes_FromStringAndSize(byte_out, byte_count); - PyMem_Free(byte_out); return out; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/mutf8/mutf8.py new/mutf8-1.1.0/mutf8/mutf8.py --- old/mutf8-1.0.6/mutf8/mutf8.py 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/mutf8/mutf8.py 2026-07-10 20:04:18.000000000 +0200 @@ -59,36 +59,30 @@ b2 = s[s_ix] b3 = s[s_ix + 1] - if b1 == 0xED and (b2 & 0xF0) == 0xA0: - # Possible six-byte codepoint. - if s_ix + 4 >= s_len: - raise UnicodeDecodeError( - 'mutf-8', - s, - s_ix - 1, - s_ix, - '3-byte or 6-byte codepoint started, but input too' - ' short to finish.' - ) - - b4 = s[s_ix + 2] + if (b1 == 0xED and (b2 & 0xF0) == 0xA0 and s_ix + 4 < s_len + and s[s_ix + 2] == 0xED and (s[s_ix + 3] & 0xF0) == 0xB0): + # Six-byte codepoint: a supplementary character written as a + # UTF-16 surrogate pair, each half encoded as its own 3-byte + # sequence. Only take this branch once we've confirmed a low + # surrogate actually follows; otherwise `b1 b2 b3` is a lone + # (unpaired) surrogate and is decoded as a normal 3-byte + # codepoint below. b5 = s[s_ix + 3] b6 = s[s_ix + 4] - - if b4 == 0xED and (b5 & 0xF0) == 0xB0: - # Definite six-byte codepoint. - s_out.append( - chr( - 0x10000 | - (b2 & 0x0F) << 0x10 | - (b3 & 0x3F) << 0x0A | - (b5 & 0x0F) << 0x06 | - (b6 & 0x3F) - ) + s_out.append( + chr( + 0x10000 + + ((b2 & 0x0F) << 0x10 | + (b3 & 0x3F) << 0x0A | + (b5 & 0x0F) << 0x06 | + (b6 & 0x3F)) ) - s_ix += 5 - continue + ) + s_ix += 5 + continue + # Regular three-byte codepoint. This also covers lone/unpaired + # surrogates, each of which is encoded as a single 3-byte sequence. s_out.append( chr( (b1 & 0x0F) << 0x0C | @@ -134,14 +128,18 @@ (0x80 | (0x3F & c)) ]) else: - # Six-byte codepoint. + # Six-byte codepoint: a supplementary character written as a + # UTF-16 surrogate pair, each half encoded as its own 3-byte + # sequence. The codepoint is first offset by 0x10000 as required + # by the surrogate-pair algorithm. + c -= 0x10000 final_string.extend([ 0xED, 0xA0 | ((c >> 0x10) & 0x0F), - 0x80 | ((c >> 0x0A) & 0x3f), + 0x80 | ((c >> 0x0A) & 0x3F), 0xED, - 0xb0 | ((c >> 0x06) & 0x0f), - 0x80 | (c & 0x3f) + 0xB0 | ((c >> 0x06) & 0x0F), + 0x80 | (c & 0x3F) ]) return bytes(final_string) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/pyproject.toml new/mutf8-1.1.0/pyproject.toml --- old/mutf8-1.0.6/pyproject.toml 1970-01-01 01:00:00.000000000 +0100 +++ new/mutf8-1.1.0/pyproject.toml 2026-07-10 20:04:18.000000000 +0200 @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "mutf8" +dynamic = ["version"] +description = "Fast MUTF-8 encoder & decoder" +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENCE"] +authors = [{ name = "Tyler Kennedy", email = "[email protected]" }] +keywords = ["mutf-8", "cesu-8", "jvm"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: C", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[project.optional-dependencies] +test = ["pytest", "pytest-benchmark"] + +[project.urls] +Homepage = "https://github.com/TkTech/mutf8"; +Source = "https://github.com/TkTech/mutf8"; +Issues = "https://github.com/TkTech/mutf8/issues"; + +[tool.setuptools] +packages = ["mutf8"] +# optional=true so a failed compile falls back to the pure-Python mutf8/mutf8.py. +ext-modules = [ + { name = "mutf8.cmutf8", sources = ["mutf8/cmutf8.c"], language = "c", optional = true }, +] + +[tool.setuptools.dynamic] +version = { attr = "mutf8.__version__" } + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = ["slow: mark test as slow to run"] + +[tool.cibuildwheel] +skip = ["*_i686"] +test-extras = ["test"] +test-command = "pytest {project}/tests" + +[tool.bumpversion] +current_version = "1.1.0" +commit = true +tag = true +tag_name = "v{new_version}" +allow_dirty = false + +[[tool.bumpversion.files]] +filename = "mutf8/__init__.py" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/setup.py new/mutf8-1.1.0/setup.py --- old/mutf8-1.0.6/setup.py 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/setup.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,43 +0,0 @@ -import os -import os.path - -from setuptools import setup, find_packages, Extension - - -root = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(root, 'README.md'), 'rb') as readme: - long_description = readme.read().decode('utf-8') - - -setup( - name='mutf8', - packages=find_packages(), - version='1.0.6', - description='Fast MUTF-8 encoder & decoder', - long_description=long_description, - long_description_content_type='text/markdown', - author='Tyler Kennedy', - author_email='[email protected]', - url='http://github.com/TkTech/mutf8', - keywords=['mutf-8', 'cesu-8', 'jvm'], - classifiers=[ - 'Programming Language :: Python :: 3', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Intended Audience :: Developers', - ], - extras_require={ - 'test': [ - 'pytest', - 'pytest-benchmark' - ] - }, - ext_modules=[ - Extension( - 'mutf8.cmutf8', - ['mutf8/cmutf8.c'], - language='c', - optional=True - ) - ] -) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/tests/test_benchmark.py new/mutf8-1.1.0/tests/test_benchmark.py --- old/mutf8-1.0.6/tests/test_benchmark.py 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/tests/test_benchmark.py 2026-07-10 20:04:18.000000000 +0200 @@ -1,28 +1,67 @@ +""" +Benchmarks comparing the pure-Python and C encoders/decoders across realistic +inputs (ASCII, CJK, emoji, mixed) and payload sizes. + +Marked ``slow``; run with ``pytest --runslow tests/test_benchmark.py``. +""" import pytest import mutf8.mutf8 as pymutf8 import mutf8.cmutf8 as cmutf8 +# Short representative units, repeated to hit each target size. Ordered from +# the most common real-world case (ASCII JVM symbols) to the rarest. +_UNITS = { + "ascii": "java/lang/Object;", # 1-byte codepoints + "cjk": "編碼測試字", # 3-byte codepoints + "emoji": "🚀🎉", # 6-byte (surrogate pair) codepoints + "mixed": "café ☕ item#42 shipped 🚚 ", # 1/2/3/6-byte mix +} + +# Target *encoded* sizes: 16b exposes per-call overhead, the rest throughput. +_SIZES = {"16b": 16, "1kb": 1024, "64kb": 64 * 1024} + + +def _scale(unit, target_bytes): + reps = max(1, round(target_bytes / len(cmutf8.encode_modified_utf8(unit)))) + return unit * reps + + +# (content, size) -> decoded str and its MUTF-8 encoding. +STR_CORPORA = { + (content, size): _scale(unit, nbytes) + for content, unit in _UNITS.items() + for size, nbytes in _SIZES.items() +} +BYTES_CORPORA = { + key: cmutf8.encode_modified_utf8(text) for key, text in STR_CORPORA.items() +} + +CORPORA = list(STR_CORPORA) +_CORPUS_IDS = [f"{content}-{size}" for content, size in CORPORA] +_IMPLS = (("py", pymutf8), ("c", cmutf8)) +_IMPL_IDS = [name for name, _ in _IMPLS] + [email protected]('group,decoder', ( - ('pymutf8', pymutf8.decode_modified_utf8), - ('cmutf8', cmutf8.decode_modified_utf8) -)) @pytest.mark.slow -def test_decode(group, decoder, benchmark): - """Compare the performance of the python and C decoders.""" - benchmark.group = 'MUTF-8 Decoding' - benchmark.extra_info['group'] = group - benchmark(decoder, b'\xED\xA0\xBD\xED\xB8\x88') [email protected]("corpus", CORPORA, ids=_CORPUS_IDS) [email protected]("impl,module", _IMPLS, ids=_IMPL_IDS) +def test_decode(corpus, impl, module, benchmark): + content, size = corpus + data = BYTES_CORPORA[corpus] + benchmark.group = f"decode {content}-{size}" + benchmark.extra_info.update(impl=impl, encoded_bytes=len(data)) + result = benchmark(module.decode_modified_utf8, data) + assert result == STR_CORPORA[corpus] [email protected]('group,encoder', ( - ('pymutf8', pymutf8.encode_modified_utf8), - ('cmutf8', cmutf8.encode_modified_utf8) -)) @pytest.mark.slow -def test_encode(group, encoder, benchmark): - """Compare the performance of the python and C encoders.""" - benchmark.group = 'MUTF-8 Encoding' - benchmark.extra_info['group'] = group - benchmark(encoder, '\U0001F608') [email protected]("corpus", CORPORA, ids=_CORPUS_IDS) [email protected]("impl,module", _IMPLS, ids=_IMPL_IDS) +def test_encode(corpus, impl, module, benchmark): + content, size = corpus + text = STR_CORPORA[corpus] + benchmark.group = f"encode {content}-{size}" + benchmark.extra_info.update(impl=impl, encoded_bytes=len(BYTES_CORPORA[corpus])) + result = benchmark(module.encode_modified_utf8, text) + assert result == BYTES_CORPORA[corpus] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/tests/test_bugs.py new/mutf8-1.1.0/tests/test_bugs.py --- old/mutf8-1.0.6/tests/test_bugs.py 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/tests/test_bugs.py 2026-07-10 20:04:18.000000000 +0200 @@ -32,3 +32,61 @@ """ str = '黑人抬棺組裝包' assert decoder(encoder(str)) == str + + +# Supplementary characters and their *correct* MUTF-8 encoding, cross-checked +# against Java's DataOutputStream.writeUTF, Android/DEX and Rust's +# residua-mutf8. See https://github.com/TkTech/mutf8/issues/5 and /issues/6. +ISSUE_5_6_VECTORS = ( + ('\U00010000', b'\xed\xa0\x80\xed\xb0\x80'), + ('\U00020000', b'\xed\xa1\x80\xed\xb0\x80'), + ('\U00040000', b'\xed\xa3\x80\xed\xb0\x80'), + ('\U00080000', b'\xed\xa7\x80\xed\xb0\x80'), + ('\U00100000', b'\xed\xaf\x80\xed\xb0\x80'), + ('\U0010FFFF', b'\xed\xaf\xbf\xed\xbf\xbf'), + ('\U00010401', b'\xed\xa0\x81\xed\xb0\x81'), # residua-mutf8 example + ('\U0001F4CB', b'\xed\xa0\xbd\xed\xb3\x8b'), # 📋 + ('\U0001F3F9', b'\xed\xa0\xbc\xed\xbf\xb9'), # 🏹 + ('\U0001F41F', b'\xed\xa0\xbd\xed\xb0\x9f'), # 🐟 +) + + +def test_issue_5_and_6(encoder, decoder): + """ + Supplementary characters (> U+FFFF) must be encoded as a UTF-16 surrogate + pair after subtracting 0x10000, matching Java/Android/CESU-8. Previously + the 0x10000 offset was omitted on encode (and compensated with a bitwise + OR on decode), producing non-standard bytes and losing data for plane 16. + + https://github.com/TkTech/mutf8/issues/5 + https://github.com/TkTech/mutf8/issues/6 + """ + for text, expected in ISSUE_5_6_VECTORS: + assert encoder(text) == expected, hex(ord(text)) + # Decoding standard MUTF-8 must recover the original codepoint. + assert decoder(expected) == text, hex(ord(text)) + # And a full round-trip must be lossless across the whole range. + assert decoder(encoder(text)) == text, hex(ord(text)) + + +def test_issue_4(decoder, encoder): + """ + A lone (unpaired) high surrogate is legal in a Java string and is encoded + as a single 3-byte sequence (0xED 0xA0-0xAF ...). The decoder must not + assume every 0xED 0xAx sequence begins a six-byte pair; when no low + surrogate follows (including at end-of-input) it must decode a 3-byte + lone surrogate instead of raising "input too short". + + https://github.com/TkTech/mutf8/issues/4 + """ + # A lone high surrogate at end-of-input (previously raised spuriously). + assert decoder(b'\xed\xa1\x80') == '\ud840' + # A lone low surrogate, likewise. + assert decoder(b'\xed\xb0\x80') == '\udc00' + # A high surrogate followed by non-surrogate data, not a pair. + assert decoder(b'\xed\xa1\x80AB') == '\ud840AB' + # Two high surrogates in a row: each is its own 3-byte codepoint. + assert decoder(b'\xed\xa1\x80\xed\xa1\x80') == '\ud840\ud840' + # Lone surrogates must survive a round-trip. + for cp in (0xD800, 0xD840, 0xDBFF, 0xDC00, 0xDFFF): + assert decoder(encoder(chr(cp))) == chr(cp), hex(cp) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mutf8-1.0.6/tests/test_modified_utf8.py new/mutf8-1.1.0/tests/test_modified_utf8.py --- old/mutf8-1.0.6/tests/test_modified_utf8.py 2021-12-29 04:00:07.000000000 +0100 +++ new/mutf8-1.1.0/tests/test_modified_utf8.py 2026-07-10 20:04:18.000000000 +0200 @@ -125,44 +125,44 @@ Test data taken from py2jdbc. """ pairs = _pairs( - (0x10000, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x80)), - (0x10001, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x81)), - (0x10002, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x82)), - (0x10004, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x84)), - (0x10008, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x88)), - (0x10010, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x90)), - (0x10020, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0xa0)), - (0x10040, (0xed, 0xa1, 0x80, 0xed, 0xb1, 0x80)), - (0x10080, (0xed, 0xa1, 0x80, 0xed, 0xb2, 0x80)), - (0x10100, (0xed, 0xa1, 0x80, 0xed, 0xb4, 0x80)), - (0x10200, (0xed, 0xa1, 0x80, 0xed, 0xb8, 0x80)), - (0x10400, (0xed, 0xa1, 0x81, 0xed, 0xb0, 0x80)), - (0x10800, (0xed, 0xa1, 0x82, 0xed, 0xb0, 0x80)), - (0x11000, (0xed, 0xa1, 0x84, 0xed, 0xb0, 0x80)), - (0x12000, (0xed, 0xa1, 0x88, 0xed, 0xb0, 0x80)), - (0x14000, (0xed, 0xa1, 0x90, 0xed, 0xb0, 0x80)), - (0x18000, (0xed, 0xa1, 0xa0, 0xed, 0xb0, 0x80)), - (0x30000, (0xed, 0xa3, 0x80, 0xed, 0xb0, 0x80)), - (0x50000, (0xed, 0xa5, 0x80, 0xed, 0xb0, 0x80)), - (0x90000, (0xed, 0xa9, 0x80, 0xed, 0xb0, 0x80)), - (0x10003, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x83)), - (0x10007, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x87)), - (0x1000f, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x8f)), - (0x1001f, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x9f)), - (0x1003f, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0xbf)), - (0x1007f, (0xed, 0xa1, 0x80, 0xed, 0xb1, 0xbf)), - (0x100ff, (0xed, 0xa1, 0x80, 0xed, 0xb3, 0xbf)), - (0x101ff, (0xed, 0xa1, 0x80, 0xed, 0xb7, 0xbf)), - (0x103ff, (0xed, 0xa1, 0x80, 0xed, 0xbf, 0xbf)), - (0x107ff, (0xed, 0xa1, 0x81, 0xed, 0xbf, 0xbf)), - (0x10fff, (0xed, 0xa1, 0x83, 0xed, 0xbf, 0xbf)), - (0x11fff, (0xed, 0xa1, 0x87, 0xed, 0xbf, 0xbf)), - (0x13fff, (0xed, 0xa1, 0x8f, 0xed, 0xbf, 0xbf)), - (0x17fff, (0xed, 0xa1, 0x9f, 0xed, 0xbf, 0xbf)), - (0x1ffff, (0xed, 0xa1, 0xbf, 0xed, 0xbf, 0xbf)), - (0x3ffff, (0xed, 0xa3, 0xbf, 0xed, 0xbf, 0xbf)), - (0x7ffff, (0xed, 0xa7, 0xbf, 0xed, 0xbf, 0xbf)), - (0xfffff, (0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf)) + (0x10000, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80)), + (0x10001, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x81)), + (0x10002, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x82)), + (0x10004, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x84)), + (0x10008, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x88)), + (0x10010, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x90)), + (0x10020, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0xa0)), + (0x10040, (0xed, 0xa0, 0x80, 0xed, 0xb1, 0x80)), + (0x10080, (0xed, 0xa0, 0x80, 0xed, 0xb2, 0x80)), + (0x10100, (0xed, 0xa0, 0x80, 0xed, 0xb4, 0x80)), + (0x10200, (0xed, 0xa0, 0x80, 0xed, 0xb8, 0x80)), + (0x10400, (0xed, 0xa0, 0x81, 0xed, 0xb0, 0x80)), + (0x10800, (0xed, 0xa0, 0x82, 0xed, 0xb0, 0x80)), + (0x11000, (0xed, 0xa0, 0x84, 0xed, 0xb0, 0x80)), + (0x12000, (0xed, 0xa0, 0x88, 0xed, 0xb0, 0x80)), + (0x14000, (0xed, 0xa0, 0x90, 0xed, 0xb0, 0x80)), + (0x18000, (0xed, 0xa0, 0xa0, 0xed, 0xb0, 0x80)), + (0x30000, (0xed, 0xa2, 0x80, 0xed, 0xb0, 0x80)), + (0x50000, (0xed, 0xa4, 0x80, 0xed, 0xb0, 0x80)), + (0x90000, (0xed, 0xa8, 0x80, 0xed, 0xb0, 0x80)), + (0x10003, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x83)), + (0x10007, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x87)), + (0x1000f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x8f)), + (0x1001f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x9f)), + (0x1003f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0xbf)), + (0x1007f, (0xed, 0xa0, 0x80, 0xed, 0xb1, 0xbf)), + (0x100ff, (0xed, 0xa0, 0x80, 0xed, 0xb3, 0xbf)), + (0x101ff, (0xed, 0xa0, 0x80, 0xed, 0xb7, 0xbf)), + (0x103ff, (0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf)), + (0x107ff, (0xed, 0xa0, 0x81, 0xed, 0xbf, 0xbf)), + (0x10fff, (0xed, 0xa0, 0x83, 0xed, 0xbf, 0xbf)), + (0x11fff, (0xed, 0xa0, 0x87, 0xed, 0xbf, 0xbf)), + (0x13fff, (0xed, 0xa0, 0x8f, 0xed, 0xbf, 0xbf)), + (0x17fff, (0xed, 0xa0, 0x9f, 0xed, 0xbf, 0xbf)), + (0x1ffff, (0xed, 0xa0, 0xbf, 0xed, 0xbf, 0xbf)), + (0x3ffff, (0xed, 0xa2, 0xbf, 0xed, 0xbf, 0xbf)), + (0x7ffff, (0xed, 0xa6, 0xbf, 0xed, 0xbf, 0xbf)), + (0xfffff, (0xed, 0xae, 0xbf, 0xed, 0xbf, 0xbf)) ) for decoded, original in pairs:
