This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-vector-index.git
The following commit(s) were added to refs/heads/main by this push:
new 01d6f58 fix: harden release binary artifacts
01d6f58 is described below
commit 01d6f5809371e3f2351aba50dd5d1c52710b3dd1
Author: JingsongLi <[email protected]>
AuthorDate: Sun Jul 26 20:01:48 2026 +0800
fix: harden release binary artifacts
---
.github/workflows/ci.yml | 36 +++-
.github/workflows/release-java.yml | 62 ++++++
.github/workflows/release-python-publish.yml | 5 +
.github/workflows/release-python.yml | 9 +
LICENSE-binary | 160 ++++++++++++++
LICENSE-binary-ffi | 107 ++++++++++
core/STORAGE_FORMAT.md | 7 +-
docs/api.html | 1 +
docs/creating-a-release.html | 2 +-
docs/development.html | 3 +-
docs/releases.html | 1 +
docs/verifying-a-release-candidate.html | 10 +-
java/pom.xml | 14 ++
.../paimon/index/vector/NativeLibraryLoader.java | 140 ++++++++++++
.../paimon/index/vector/VectorIndexNative.java | 4 +
.../index/vector/VectorIndexJavaApiTest.java | 24 +++
.../vector/VectorIndexNativeHandleSafetyTest.java | 6 +-
.../vector/VectorIndexNativeLoaderSmokeTest.java | 49 +++++
.../vector/VectorIndexNativePanicBoundaryTest.java | 6 +-
.../vector/VectorIndexNativeValidationTest.java | 6 +-
python/pyproject.toml | 8 +-
python/setup.py | 56 ++++-
tools/README.md | 17 +-
tools/check_license_headers.py | 2 +
tools/create_source_archive.sh | 44 ++++
tools/create_source_release.sh | 10 +-
tools/deploy_java_staging.sh | 56 ++++-
tools/verify_binary_artifact.py | 237 +++++++++++++++++++++
28 files changed, 1036 insertions(+), 46 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 064a8d9..e8b61d8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,7 +46,10 @@ jobs:
python-version: '3.9'
- name: Check benchmark tools on Python 3.9
- run: python3 -m py_compile tools/convert_ann_benchmarks.py
+ run: |
+ python3 -m py_compile \
+ tools/convert_ann_benchmarks.py \
+ tools/verify_binary_artifact.py
- name: Validate .asf.yaml
run: pip install pyyaml -q && python3 tools/validate_asf_yaml.py
@@ -54,6 +57,19 @@ jobs:
- name: Check ASF license headers
run: python3 tools/check_license_headers.py
+ - name: Check binary legal inventory
+ run: python3 tools/verify_binary_artifact.py --source
+
+ - name: Check reproducible source archive
+ shell: bash
+ run: |
+ set -euo pipefail
+ first="$(mktemp)"
+ second="$(mktemp)"
+ tools/create_source_archive.sh 0.0.0 "$first"
+ tools/create_source_archive.sh 0.0.0 "$second"
+ cmp "$first" "$second"
+
- name: Install cargo-deny
uses: taiki-e/install-action@v2
with:
@@ -192,6 +208,21 @@ jobs:
env:
RAYON_NUM_THREADS: 4
+ - name: Package and test the bundled JNI loader
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p java/src/main/resources/native/linux/x86_64
+ cp target/release/libpaimon_vindex_jni.so \
+ java/src/main/resources/native/linux/x86_64/
+ mvn -f java/pom.xml package -DskipTests
+ jar_file="java/target/paimon-vector-index-java-$(sed -n
's#.*<version>\([^<]*\)</version>.*#\1#p' java/pom.xml | sed -n '2p').jar"
+ python3 tools/verify_binary_artifact.py \
+ --jar "$jar_file" \
+ --jar-native native/linux/x86_64/libpaimon_vindex_jni.so
+ java -cp "$jar_file:java/target/test-classes" \
+ org.apache.paimon.index.vector.VectorIndexNativeLoaderSmokeTest
+
python-build:
runs-on: ubuntu-latest
steps:
@@ -258,6 +289,9 @@ jobs:
python -m pip install --upgrade pip build
python -m build --wheel
+ - name: Verify wheel legal and native contents
+ run: python tools/verify_binary_artifact.py --wheel "python/dist/*.whl"
+
- name: Test wheel
if: runner.os != 'Windows'
working-directory: python
diff --git a/.github/workflows/release-java.yml
b/.github/workflows/release-java.yml
index ecf7f10..d05dae0 100644
--- a/.github/workflows/release-java.yml
+++ b/.github/workflows/release-java.yml
@@ -167,8 +167,70 @@ jobs:
working-directory: java
run: mvn clean verify -Prelease -Dgpg.skip=true -DskipTests
+ - name: Verify multi-platform Java package
+ shell: bash
+ run: |
+ set -euo pipefail
+ jar_file="java/target/paimon-vector-index-java-$(sed -n
's#.*<version>\([^<]*\)</version>.*#\1#p' java/pom.xml | sed -n '2p').jar"
+ sources_jar="${jar_file%.jar}-sources.jar"
+ python3 tools/verify_binary_artifact.py --jar "$jar_file"
+ if grep -Eq '^(native/|META-INF/LICENSE-binary$)' <<<"$(jar tf
"$sources_jar")"; then
+ echo "Sources JAR contains binary-only resources" >&2
+ exit 1
+ fi
+ java -cp "$jar_file:java/target/test-classes" \
+ org.apache.paimon.index.vector.VectorIndexNativeLoaderSmokeTest
+
- name: Upload Java package
uses: actions/upload-artifact@v5
with:
name: java-package
path: java/target/*.jar
+
+ smoke-java-package:
+ if: github.repository == 'apache/paimon-vector-index' &&
startsWith(github.ref, 'refs/tags/') && contains(github.ref_name, '-rc')
+ needs: [package-java]
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-latest
+ classpath_separator: ':'
+ - os: ubuntu-24.04-arm
+ classpath_separator: ':'
+ - os: macos-latest
+ classpath_separator: ':'
+ - os: windows-latest
+ classpath_separator: ';'
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Set up JDK ${{ env.JDK_VERSION }}
+ uses: actions/setup-java@v4
+ with:
+ java-version: ${{ env.JDK_VERSION }}
+ distribution: 'temurin'
+
+ - name: Download final Java package
+ uses: actions/download-artifact@v5
+ with:
+ name: java-package
+ path: java-package
+
+ - name: Load the bundled JNI library from the final JAR
+ shell: bash
+ run: |
+ set -euo pipefail
+ jar_file="$(
+ find java-package -type f -name '*.jar' \
+ ! -name '*-sources.jar' \
+ ! -name '*-javadoc.jar' \
+ -print -quit
+ )"
+ test -n "$jar_file"
+ mkdir -p smoke-classes
+ javac -cp "$jar_file" -d smoke-classes \
+
java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.java
+ java -cp "$jar_file${{ matrix.classpath_separator }}smoke-classes" \
+ org.apache.paimon.index.vector.VectorIndexNativeLoaderSmokeTest
diff --git a/.github/workflows/release-python-publish.yml
b/.github/workflows/release-python-publish.yml
index 44b1cc8..ef8ad94 100644
--- a/.github/workflows/release-python-publish.yml
+++ b/.github/workflows/release-python-publish.yml
@@ -37,12 +37,17 @@ jobs:
if: github.repository == 'apache/paimon-vector-index' &&
startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
steps:
+ - uses: actions/checkout@v6
+
- uses: actions/download-artifact@v5
with:
pattern: wheels-*
merge-multiple: true
path: dist
+ - name: Verify wheel legal and native contents
+ run: python3 tools/verify_binary_artifact.py --wheel "dist/*.whl"
+
- name: Verify wheel versions
env:
TAG_NAME: ${{ github.ref_name }}
diff --git a/.github/workflows/release-python.yml
b/.github/workflows/release-python.yml
index 610de65..0ce8ff6 100644
--- a/.github/workflows/release-python.yml
+++ b/.github/workflows/release-python.yml
@@ -75,6 +75,9 @@ jobs:
cargo build --release -p paimon-vindex-ffi &&
cp target/release/libpaimon_vindex_ffi.so {package}/paimon_vindex/
+ - name: Verify wheel legal and native contents
+ run: python3 tools/verify_binary_artifact.py --wheel "wheelhouse/*.whl"
+
- name: Upload wheels
uses: actions/upload-artifact@v5
with:
@@ -133,6 +136,9 @@ jobs:
rm python/dist/*.whl
mv python/dist/repaired/*.whl python/dist/
+ - name: Verify wheel legal and native contents
+ run: python3 tools/verify_binary_artifact.py --wheel
"python/dist/*.whl"
+
- name: Upload wheel
uses: actions/upload-artifact@v5
with:
@@ -176,6 +182,9 @@ jobs:
working-directory: python
run: python -m build --wheel
+ - name: Verify wheel legal and native contents
+ run: python tools/verify_binary_artifact.py --wheel "python/dist/*.whl"
+
- name: Upload wheel
uses: actions/upload-artifact@v5
with:
diff --git a/LICENSE-binary b/LICENSE-binary
new file mode 100644
index 0000000..f9bb069
--- /dev/null
+++ b/LICENSE-binary
@@ -0,0 +1,160 @@
+Apache Paimon Vector Index binary distributions
+================================================
+
+The Java binary distribution includes compiled Rust dependencies.
+
+Except for the components listed below, Apache Paimon Vector Index selects the
+Apache License, Version 2.0 option for dependencies that offer it. The full
+Apache License, Version 2.0 text is in LICENSE.
+
+byteorder 1.5.0 and memchr 2.8.1
+--------------------------------
+
+These components are distributed under the MIT license option.
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Andrew Gallant
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+bytes 1.11.1
+------------
+
+Copyright (c) 2018 Carl Lerche
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+combine 4.6.7
+-------------
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Markus Westerlind
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+crunchy 0.2.4
+-------------
+
+The MIT License (MIT)
+
+Copyright 2017-2023 Eira Fransham.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+unicode-ident 1.0.24
+--------------------
+
+The Rust code is distributed under the Apache License, Version 2.0 option. Its
+Unicode data is additionally covered by the following Unicode License V3.
+
+UNICODE LICENSE V3
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright © 1991-2023 Unicode, Inc.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
diff --git a/LICENSE-binary-ffi b/LICENSE-binary-ffi
new file mode 100644
index 0000000..71de11a
--- /dev/null
+++ b/LICENSE-binary-ffi
@@ -0,0 +1,107 @@
+Apache Paimon Vector Index Python binary distributions
+=======================================================
+
+The Python wheels include the compiled paimon-vindex-ffi Rust dependency
+closure.
+
+Except for the components listed below, Apache Paimon Vector Index selects the
+Apache License, Version 2.0 option for dependencies that offer it. The full
+Apache License, Version 2.0 text is in LICENSE.
+
+byteorder 1.5.0
+---------------
+
+This component is distributed under the MIT license option.
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Andrew Gallant
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+crunchy 0.2.4
+-------------
+
+The MIT License (MIT)
+
+Copyright 2017-2023 Eira Fransham.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+unicode-ident 1.0.24
+--------------------
+
+The Rust code is distributed under the Apache License, Version 2.0 option. Its
+Unicode data is additionally covered by the following Unicode License V3.
+
+UNICODE LICENSE V3
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright © 1991-2023 Unicode, Inc.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
diff --git a/core/STORAGE_FORMAT.md b/core/STORAGE_FORMAT.md
index 9616d59..27ec901 100644
--- a/core/STORAGE_FORMAT.md
+++ b/core/STORAGE_FORMAT.md
@@ -20,8 +20,11 @@
# Vector Index Storage Format
This document describes the v1 on-disk formats written by the
-`paimon-vindex-core` crate. Version 1 is the first release format. Pre-release
-layouts are not part of the compatibility contract.
+`paimon-vindex-core` crate starting with version 0.3.0. The 0.3.0 release
+intentionally resets the pre-1.0 storage contract: it does not read the
+experimental IVF-HNSW-FLAT (`IHFL`) or IVF-HNSW-SQ (`IHSQ`) layouts published
+by 0.2.x. Rebuild those indexes when upgrading, and do not rely on 0.2.x
+readers as a rollback path for files written by 0.3.0.
## Compatibility Policy
diff --git a/docs/api.html b/docs/api.html
index 1940e72..0fdde75 100644
--- a/docs/api.html
+++ b/docs/api.html
@@ -201,6 +201,7 @@ auto result = reader.search(
<section class="article-section" id="java">
<h2>Java / JNI</h2>
<p>The package is <code>org.apache.paimon.index.vector</code>.
String options map directly to Paimon table and index properties; Rust parses
and validates them when a Trainer is created.</p>
+ <p>The Maven JAR contains the supported Linux x86-64, Linux aarch64,
macOS arm64, and Windows x86-64 JNI libraries. The first native API call
selects the current platform, extracts that library to a private temporary
file, and loads it automatically. To use a separately built library instead,
start the JVM with
<code>-Dpaimon.vindex.native.path=/absolute/path/to/the/library</code>.</p>
<div class="code-block"><span class="code-label">Java · build and
search</span><pre><code>Map<String, String> options = new
HashMap<>();
options.put("index.type", "ivf_sq");
options.put("metric", "l2");
diff --git a/docs/creating-a-release.html b/docs/creating-a-release.html
index a16f397..ce41ef1 100644
--- a/docs/creating-a-release.html
+++ b/docs/creating-a-release.html
@@ -137,7 +137,7 @@ cp
tools/release/apache-paimon-vector-index-${RELEASE_VERSION}-src.* \
svn add "paimon-dist-dev/paimon-vector-index-${RELEASE_VERSION}-rc${RC_NUM}"
svn commit -m "Add Paimon Vector Index ${RELEASE_VERSION} RC${RC_NUM}" \
paimon-dist-dev</code></pre></div>
- <p>The script archives committed Git objects, excludes
development-only files, writes a reproducible gzip stream, creates an armored
detached signature, and records a SHA-512 checksum.</p>
+ <p>The script archives the current commit with its stable commit
timestamp, excludes development-only files, writes a reproducible gzip stream,
creates an armored detached signature, and records a SHA-512 checksum.
Re-running it for the same commit and release version must produce the same
<code>.tgz</code> bytes.</p>
<h3>Stage the Java convenience artifact</h3>
<p>Find the numeric run ID of the successful RC-tag
<code>Release</code> workflow, then validate the exact CI native artifacts
before any remote deploy.</p>
diff --git a/docs/development.html b/docs/development.html
index 6383e8d..6993a82 100644
--- a/docs/development.html
+++ b/docs/development.html
@@ -154,7 +154,7 @@ LD_LIBRARY_PATH=target/release
cpp/build/test_vindex_cpp</code></pre></div>
<section class="article-section" id="java">
<h2>Java / JNI tests</h2>
- <p>The Maven test phase compiles the Java sources and runs
<code>VectorIndexJavaApiTest</code>. This test covers result and metadata
objects, closed-handle behavior, and API compilation without loading the native
library:</p>
+ <p>The Maven test phase compiles the Java sources and runs
<code>VectorIndexJavaApiTest</code>. This test covers result and metadata
objects, native resource mapping, closed-handle behavior, and API compilation
without loading the native library:</p>
<div class="code-block"><span
class="code-label">Shell</span><pre><code>mvn -f java/pom.xml
test</code></pre></div>
<p>Native validation, panic-boundary, and handle-safety tests are
standalone <code>main</code> programs. The following Linux commands mirror the
additional JNI checks run by CI:</p>
<div class="code-block"><span class="code-label">Shell · Linux JNI
checks</span><pre><code>cargo build -p paimon-vindex-jni --release
@@ -169,6 +169,7 @@ java -cp java/target/test-classes:java/target/classes \
org.apache.paimon.index.vector.VectorIndexNativeHandleSafetyTest \
"$(pwd)/target/release/libpaimon_vindex_jni.so"</code></pre></div>
<div class="callout"><strong>Platform note</strong>The
shared-library filename above is for Linux. Use the native library filename
produced in <code>target/release</code> on macOS or Windows.</div>
+ <p>Release CI additionally builds the multi-platform Maven JAR and
runs <code>VectorIndexNativeLoaderSmokeTest</code> from the final
<code>java-package</code> artifact with no native path argument on Linux
x86-64, Linux aarch64, macOS arm64, and Windows x86-64. This ensures consumers
can use every bundled JNI library directly.</p>
</section>
<section class="article-section" id="python">
diff --git a/docs/releases.html b/docs/releases.html
index 1e34440..aed9f95 100644
--- a/docs/releases.html
+++ b/docs/releases.html
@@ -49,6 +49,7 @@
<section class="article-section" id="upcoming">
<h2>Upcoming: 0.3.0</h2>
<p>The repository is currently developing the 0.3.0 line. Until an
ASF vote passes and the signed source archive appears under Apache downloads,
code and packages from this line are development artifacts rather than an
Apache release.</p>
+ <div class="callout warning"><strong>Stored-index upgrade
note</strong>Version 0.3.0 intentionally does not read the experimental
IVF-HNSW-FLAT (<code>IHFL</code>) and IVF-HNSW-SQ (<code>IHSQ</code>) files
written by 0.2.x. Rebuild those indexes with a 0.3.0-supported index type
during upgrade. Index files written by 0.3.0 are likewise not a safe rollback
boundary for 0.2.x; retain source vectors or a 0.2-compatible index copy until
the upgrade is accepted.</div>
</section>
<section class="article-section" id="release-020">
diff --git a/docs/verifying-a-release-candidate.html
b/docs/verifying-a-release-candidate.html
index 72473d8..a3648f1 100644
--- a/docs/verifying-a-release-candidate.html
+++ b/docs/verifying-a-release-candidate.html
@@ -155,8 +155,12 @@ paimon-vindex-core = {
# native/linux/x86_64/libpaimon_vindex_jni.so
# native/linux/aarch64/libpaimon_vindex_jni.so
# native/macos/aarch64/libpaimon_vindex_jni.dylib
-# native/windows/x86_64/paimon_vindex_jni.dll</code></pre></div>
- <p>Verify the main JAR plus source and Javadoc JARs, POM,
signatures, and checksums. Confirm that the source JAR does not embed native
libraries and that the POM version has no <code>-SNAPSHOT</code> suffix.</p>
+# native/windows/x86_64/paimon_vindex_jni.dll
+# META-INF/LICENSE
+# META-INF/NOTICE
+# META-INF/LICENSE-binary
+# org/apache/paimon/index/vector/NativeLibraryLoader.class</code></pre></div>
+ <p>Run the smoke program with the staged main JAR first on the
classpath and without passing or preloading a native library. Verify that it
creates a native Trainer successfully. Also verify the main JAR plus source and
Javadoc JARs, POM, signatures, and checksums. Confirm that the source JAR does
not embed native libraries and that the POM version has no
<code>-SNAPSHOT</code> suffix.</p>
<h3>Python TestPyPI</h3>
<div class="code-block"><span class="code-label">Shell · isolated
environment</span><pre><code>python3 -m venv rc-wheel
@@ -166,7 +170,7 @@ python -m pip install \
--extra-index-url https://pypi.org/simple/ \
"paimon-vindex==${RELEASE_VERSION}rc${RC_NUM}"
python -c "import paimon_vindex; print('OK')"</code></pre></div>
- <p>Check that TestPyPI provides Linux x86-64, Linux aarch64, macOS
arm64, and Windows x86-64 wheels, and test the wheel matching your platform
against a small build/search/read round trip.</p>
+ <p>Check that TestPyPI provides Linux x86-64, Linux aarch64, macOS
arm64, and Windows x86-64 wheels. Every wheel must contain
<code>LICENSE</code>, <code>NOTICE</code>, and <code>LICENSE-binary</code>
alongside the package and in the wheel metadata license directory. Test the
wheel matching your platform against a small build/search/read round trip.</p>
</section>
<section class="article-section" id="vote">
diff --git a/java/pom.xml b/java/pom.xml
index 590c8de..6d0e1a4 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -70,6 +70,18 @@
</properties>
<build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>${project.basedir}/..</directory>
+ <targetPath>META-INF</targetPath>
+ <includes>
+ <include>LICENSE-binary</include>
+ </includes>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -139,6 +151,8 @@
code (and META-INF metadata), not platform
binaries. -->
<excludes>
<exclude>native/**</exclude>
+ <exclude>LICENSE-binary</exclude>
+ <exclude>**/LICENSE-binary</exclude>
</excludes>
</configuration>
<executions>
diff --git
a/java/src/main/java/org/apache/paimon/index/vector/NativeLibraryLoader.java
b/java/src/main/java/org/apache/paimon/index/vector/NativeLibraryLoader.java
new file mode 100644
index 0000000..51cfb95
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/vector/NativeLibraryLoader.java
@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.paimon.index.vector;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Locale;
+
+/** Loads the JNI library from an explicit path or the platform resource
bundled in the JAR. */
+final class NativeLibraryLoader {
+
+ static final String NATIVE_PATH_PROPERTY = "paimon.vindex.native.path";
+
+ private static boolean loaded;
+
+ private NativeLibraryLoader() {}
+
+ static synchronized void load() {
+ if (loaded) {
+ return;
+ }
+
+ String explicitPath = System.getProperty(NATIVE_PATH_PROPERTY);
+ if (explicitPath != null && !explicitPath.trim().isEmpty()) {
+
System.load(Paths.get(explicitPath).toAbsolutePath().normalize().toString());
+ loaded = true;
+ return;
+ }
+
+ String resourcePath =
+ resourcePath(
+ System.getProperty("os.name"),
System.getProperty("os.arch"));
+ String fileName = resourcePath.substring(resourcePath.lastIndexOf('/')
+ 1);
+ int extension = fileName.lastIndexOf('.');
+ String suffix = extension >= 0 ? fileName.substring(extension) : null;
+
+ try (InputStream input =
NativeLibraryLoader.class.getResourceAsStream(resourcePath)) {
+ if (input == null) {
+ throw loadError(
+ "Bundled JNI library is missing: "
+ + resourcePath
+ + ". Set -D"
+ + NATIVE_PATH_PROPERTY
+ + "=/absolute/path/to/the/library to use an
external build.",
+ null);
+ }
+
+ Path extracted = Files.createTempFile("paimon-vindex-jni-",
suffix);
+ extracted.toFile().deleteOnExit();
+ Files.copy(input, extracted, StandardCopyOption.REPLACE_EXISTING);
+ System.load(extracted.toAbsolutePath().toString());
+ loaded = true;
+ } catch (IOException e) {
+ throw loadError("Failed to extract bundled JNI library " +
resourcePath, e);
+ }
+ }
+
+ static String resourcePath(String osName, String osArch) {
+ String os = normalizeOs(osName);
+ String arch = normalizeArch(osArch);
+ String fileName;
+
+ if ("linux".equals(os)) {
+ fileName = "libpaimon_vindex_jni.so";
+ } else if ("macos".equals(os)) {
+ fileName = "libpaimon_vindex_jni.dylib";
+ } else if ("windows".equals(os)) {
+ fileName = "paimon_vindex_jni.dll";
+ } else {
+ throw loadError("Unsupported operating system: " + osName, null);
+ }
+
+ if (("macos".equals(os) && !"aarch64".equals(arch))
+ || ("windows".equals(os) && !"x86_64".equals(arch))) {
+ throw loadError(
+ "No bundled JNI library for "
+ + osName
+ + " / "
+ + osArch
+ + ". Set -D"
+ + NATIVE_PATH_PROPERTY
+ + "=/absolute/path/to/the/library to use an
external build.",
+ null);
+ }
+
+ return "/native/" + os + "/" + arch + "/" + fileName;
+ }
+
+ private static String normalizeOs(String osName) {
+ String value = osName == null ? "" : osName.toLowerCase(Locale.ROOT);
+ if (value.contains("linux")) {
+ return "linux";
+ }
+ if (value.contains("mac") || value.contains("darwin")) {
+ return "macos";
+ }
+ if (value.contains("windows")) {
+ return "windows";
+ }
+ return value;
+ }
+
+ private static String normalizeArch(String osArch) {
+ String value = osArch == null ? "" : osArch.toLowerCase(Locale.ROOT);
+ if ("amd64".equals(value) || "x86_64".equals(value) ||
"x64".equals(value)) {
+ return "x86_64";
+ }
+ if ("aarch64".equals(value) || "arm64".equals(value)) {
+ return "aarch64";
+ }
+ throw loadError("Unsupported CPU architecture: " + osArch, null);
+ }
+
+ private static UnsatisfiedLinkError loadError(String message, Throwable
cause) {
+ UnsatisfiedLinkError error = new UnsatisfiedLinkError(message);
+ if (cause != null) {
+ error.initCause(cause);
+ }
+ return error;
+ }
+}
diff --git
a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
index cbbd0a9..7d48be3 100644
--- a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
+++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
@@ -19,6 +19,10 @@ package org.apache.paimon.index.vector;
final class VectorIndexNative {
+ static {
+ NativeLibraryLoader.load();
+ }
+
private VectorIndexNative() {}
static native long createTrainer(String[] optionKeys, String[]
optionValues);
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
index 3e86508..39f6ad9 100644
---
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
@@ -24,6 +24,7 @@ import java.util.Map;
public class VectorIndexJavaApiTest {
public static void main(String[] args) {
+ testNativeLibraryResourcePaths();
testSingleResultCopiesArrays();
testBatchResultCopiesArraysAndSlicesRows();
testMetadata();
@@ -36,6 +37,29 @@ public class VectorIndexJavaApiTest {
testReaderAndWriterApiCompile();
}
+ private static void testNativeLibraryResourcePaths() {
+ assertEquals(
+ "/native/linux/x86_64/libpaimon_vindex_jni.so",
+ NativeLibraryLoader.resourcePath("Linux", "amd64"));
+ assertEquals(
+ "/native/linux/aarch64/libpaimon_vindex_jni.so",
+ NativeLibraryLoader.resourcePath("Linux", "arm64"));
+ assertEquals(
+ "/native/macos/aarch64/libpaimon_vindex_jni.dylib",
+ NativeLibraryLoader.resourcePath("Mac OS X", "aarch64"));
+ assertEquals(
+ "/native/windows/x86_64/paimon_vindex_jni.dll",
+ NativeLibraryLoader.resourcePath("Windows 11", "x86_64"));
+ assertThrows(
+ UnsatisfiedLinkError.class,
+ new ThrowingRunnable() {
+ @Override
+ public void run() {
+ NativeLibraryLoader.resourcePath("Mac OS X", "x86_64");
+ }
+ });
+ }
+
private static void testSearchParametersRemainAlgorithmSpecific() {
VectorSearchParams params = new VectorSearchParams(10,
4).withLSearch(200);
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java
index ac348b9..8f8d19d 100644
---
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java
@@ -26,11 +26,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class VectorIndexNativeHandleSafetyTest {
public static void main(String[] args) {
- if (args.length != 1) {
- throw new IllegalArgumentException("native library path is
required");
- }
-
- System.load(args[0]);
+ VectorIndexNativeLoaderSmokeTest.configureExternalLibrary(args);
testWriterRejectsReentrantCloseDuringNativeCall();
testReaderRejectsReentrantCloseDuringNativeCall();
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.java
new file mode 100644
index 0000000..450468a
--- /dev/null
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.paimon.index.vector;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** Smoke test used with the packaged multi-platform JAR as the first
classpath entry. */
+public class VectorIndexNativeLoaderSmokeTest {
+
+ public static void main(String[] args) {
+ configureExternalLibrary(args);
+
+ Map<String, String> options = new HashMap<>();
+ options.put("index.type", "ivf_flat");
+ options.put("dimension", "2");
+ options.put("nlist", "1");
+ options.put("metric", "l2");
+
+ try (VectorIndexTrainer ignored = VectorIndexTrainer.create(options)) {
+ // Creating the native trainer proves that the JNI methods are
linked.
+ }
+ }
+
+ static void configureExternalLibrary(String[] args) {
+ if (args.length > 1) {
+ throw new IllegalArgumentException(
+ "expected zero arguments or one native library path");
+ }
+ if (args.length == 1) {
+ System.setProperty(NativeLibraryLoader.NATIVE_PATH_PROPERTY,
args[0]);
+ }
+ }
+}
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java
index ec0273f..2d0e9b8 100644
---
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java
@@ -24,11 +24,7 @@ import java.util.Map;
public class VectorIndexNativePanicBoundaryTest {
public static void main(String[] args) {
- if (args.length != 1) {
- throw new IllegalArgumentException("native library path is
required");
- }
-
- System.load(args[0]);
+ VectorIndexNativeLoaderSmokeTest.configureExternalLibrary(args);
testVoidEntrypointErrorBecomesRuntimeException();
testObjectEntrypointPanicBecomesRuntimeException();
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
index f872df0..06cbff9 100644
---
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
@@ -29,11 +29,7 @@ public class VectorIndexNativeValidationTest {
private static final int ROUNDTRIP_VECTOR_COUNT = ROUNDTRIP_NLIST *
ROUNDTRIP_PER_LIST;
public static void main(String[] args) {
- if (args.length != 1) {
- throw new IllegalArgumentException("native library path is
required");
- }
-
- System.load(args[0]);
+ VectorIndexNativeLoaderSmokeTest.configureExternalLibrary(args);
testWriterValidationComesFromCore();
testWriterRejectsNonFiniteValues();
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 788c8fc..4b5e0fe 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -16,14 +16,15 @@
# under the License.
[build-system]
-requires = ["setuptools>=64", "wheel"]
+requires = ["setuptools>=77", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "paimon-vindex"
version = "0.4.0"
description = "Python bindings for Apache Paimon Vector Index"
-license = {text = "Apache-2.0"}
+license = "Apache-2.0"
+license-files = ["LICENSE", "NOTICE", "LICENSE-binary"]
requires-python = ">=3.9"
dependencies = ["numpy"]
@@ -40,6 +41,9 @@ include = ["paimon_vindex*"]
[tool.setuptools.package-data]
paimon_vindex = [
+ "LICENSE",
+ "NOTICE",
+ "LICENSE-binary",
"libpaimon_vindex_ffi.dylib",
"libpaimon_vindex_ffi.so",
"paimon_vindex_ffi.dll",
diff --git a/python/setup.py b/python/setup.py
index 85a706f..81940a5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-"""Build helper: copies the pre-built native FFI library into the package."""
+"""Build helper: stages the native library and binary-distribution legal
files."""
import os
import platform
@@ -25,6 +25,13 @@ from setuptools import Distribution, setup
from setuptools.command.build_py import build_py
from wheel.bdist_wheel import bdist_wheel
+LEGAL_FILES = ("LICENSE", "NOTICE", "LICENSE-binary")
+LEGAL_SOURCE_FILES = {
+ "LICENSE": "LICENSE",
+ "NOTICE": "NOTICE",
+ "LICENSE-binary": "LICENSE-binary-ffi",
+}
+
def _lib_name():
system = platform.system()
@@ -55,6 +62,31 @@ def _find_native_lib():
return None
+def _stage_legal_files(destination_dir):
+ here = os.path.dirname(os.path.abspath(__file__))
+ staged = []
+
+ for name in LEGAL_FILES:
+ source = os.path.join(here, "..", LEGAL_SOURCE_FILES[name])
+ destination = os.path.join(destination_dir, name)
+ if not os.path.isfile(source):
+ raise RuntimeError(f"required binary legal file is missing:
{source}")
+ if os.path.exists(destination):
+ with open(source, "rb") as source_file:
+ source_bytes = source_file.read()
+ with open(destination, "rb") as destination_file:
+ destination_bytes = destination_file.read()
+ if source_bytes != destination_bytes:
+ raise RuntimeError(
+ f"staged binary legal file does not match repository copy:
{destination}"
+ )
+ continue
+ shutil.copy2(source, destination)
+ staged.append(destination)
+
+ return staged
+
+
class BuildPyWithNativeLib(build_py):
def run(self):
src = _find_native_lib()
@@ -65,7 +97,14 @@ class BuildPyWithNativeLib(build_py):
_lib_name(),
)
shutil.copy2(src, dst)
- super().run()
+ staged_legal_files = _stage_legal_files(
+ os.path.join(os.path.dirname(os.path.abspath(__file__)),
"paimon_vindex")
+ )
+ try:
+ super().run()
+ finally:
+ for path in staged_legal_files:
+ os.remove(path)
class PlatformWheel(bdist_wheel):
@@ -87,7 +126,14 @@ class BinaryDistribution(Distribution):
return True
-setup(
- cmdclass={"build_py": BuildPyWithNativeLib, "bdist_wheel": PlatformWheel},
- distclass=BinaryDistribution,
+staged_distribution_legal_files = _stage_legal_files(
+ os.path.dirname(os.path.abspath(__file__))
)
+try:
+ setup(
+ cmdclass={"build_py": BuildPyWithNativeLib, "bdist_wheel":
PlatformWheel},
+ distclass=BinaryDistribution,
+ )
+finally:
+ for path in staged_distribution_legal_files:
+ os.remove(path)
diff --git a/tools/README.md b/tools/README.md
index 95154e2..17cac1c 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -62,13 +62,17 @@ GitHub Actions does **not** sign or deploy the Java staging
artifacts. The
release workflow only:
1. builds the four JNI native libraries;
-2. verifies the Java release profile with GPG disabled; and
-3. uploads the native libraries and verified Java jars as workflow artifacts.
+2. packages the multi-platform JAR and smoke-tests the final JAR without an
+ external native path on all four supported platform/architecture pairs; and
+3. uploads the native libraries plus the verified `java-package` JARs as
+ workflow artifacts.
The committer then runs this script locally. The script checks that the release
workflow run succeeded for the current RC tag, downloads the native libraries,
-verifies their platform formats, places them into the Java resource tree, and
-runs Maven locally.
+and `java-package`, verifies their platform formats and legal files, places the
+native libraries into the Java resource tree, and runs Maven locally. Both the
+CI-generated JAR and the locally staged JAR must pass the bundled-native loader
+smoke test before deployment.
### Required local setup
@@ -184,7 +188,7 @@ Always run a dry-run first with the real RC workflow
artifacts:
```
Dry-run mode validates the GitHub Actions run id, downloads the native
-libraries, and runs:
+libraries and `java-package`, and runs:
```bash
mvn clean verify -Prelease -Dgpg.skip=true -DskipTests
@@ -199,6 +203,9 @@ It does not sign and does not deploy to Nexus. It verifies:
commit matches the RC tag;
- all four native libraries are present;
- native library file formats match their target platforms;
+- both the CI-generated and locally packaged JARs contain binary legal files,
+ the four native libraries, and `NativeLibraryLoader`;
+- both JARs load the current platform library without an external native path;
- the Java jar, sources jar, and javadoc jar are produced;
- the Java jar contains all four native library entries.
diff --git a/tools/check_license_headers.py b/tools/check_license_headers.py
index c959926..b22fdc4 100755
--- a/tools/check_license_headers.py
+++ b/tools/check_license_headers.py
@@ -36,6 +36,8 @@ EXEMPT_FILES = {
"ffi/DEPENDENCIES.rust.tsv",
"jni/DEPENDENCIES.rust.tsv",
"LICENSE",
+ "LICENSE-binary",
+ "LICENSE-binary-ffi",
"NOTICE",
# Golden test data; adding comments changes the fixture format.
"core/tests/fixtures/ivf_flat_v1.hex",
diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh
new file mode 100755
index 0000000..bb0f3a2
--- /dev/null
+++ b/tools/create_source_archive.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ $# -ne 2 || -z "$1" || -z "$2" ]]; then
+ echo "Usage: create_source_archive.sh RELEASE_VERSION OUTPUT_FILE" >&2
+ exit 1
+fi
+
+RELEASE_VERSION=$1
+OUTPUT_FILE=$2
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+REPO_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
+
+# Archive the commit, rather than its tree object, so Git uses the stable
+# commit timestamp instead of the current time for tar headers.
+git -C "$REPO_DIR" archive \
+ --format=tar \
+ --prefix="paimon-vector-index-${RELEASE_VERSION}/" \
+ HEAD . \
+ ':(exclude).gitignore' ':(exclude).gitattributes' \
+ ':(exclude).asf.yaml' ':(exclude).github' \
+ ':(exclude)deploysettings.xml' ':(exclude)target' \
+ ':(exclude).idea' ':(exclude)*.iml' ':(exclude).DS_Store' \
+ | gzip -n > "$OUTPUT_FILE"
diff --git a/tools/create_source_release.sh b/tools/create_source_release.sh
index cc04991..b437db4 100755
--- a/tools/create_source_release.sh
+++ b/tools/create_source_release.sh
@@ -64,13 +64,9 @@ cd ..
echo "Creating source package"
ARCHIVE="apache-paimon-vector-index-${RELEASE_VERSION}-src.tgz"
-# Archive from Git objects so filesystem metadata such as macOS xattrs is not
included.
-git archive --format=tar --prefix="paimon-vector-index-${RELEASE_VERSION}/"
'HEAD^{tree}' . \
- ':(exclude).gitignore' ':(exclude).gitattributes' \
- ':(exclude).asf.yaml' ':(exclude).github' \
- ':(exclude)deploysettings.xml' ':(exclude)target' \
- ':(exclude).idea' ':(exclude)*.iml' ':(exclude).DS_Store' \
- | gzip -n > "tools/release/${ARCHIVE}"
+"${CURR_DIR}/create_source_archive.sh" \
+ "${RELEASE_VERSION}" \
+ "${CURR_DIR}/release/${ARCHIVE}"
cd tools/release
diff --git a/tools/deploy_java_staging.sh b/tools/deploy_java_staging.sh
index 201106a..0795a94 100755
--- a/tools/deploy_java_staging.sh
+++ b/tools/deploy_java_staging.sh
@@ -225,7 +225,7 @@ else
fi
check_java_package_inputs_clean() {
- local paths=(java DEPENDENCIES.rust.tsv)
+ local paths=(java DEPENDENCIES.rust.tsv LICENSE-binary
tools/verify_binary_artifact.py)
local untracked
if ! git -C "$REPO_DIR" diff --quiet -- "${paths[@]}" ||
@@ -309,7 +309,8 @@ for artifact in \
native-linux-x86_64 \
native-linux-aarch64 \
native-macos-aarch64 \
- native-windows-x86_64
+ native-windows-x86_64 \
+ java-package
do
gh run download "$RUN_ID" \
--repo "$REPO" \
@@ -491,16 +492,34 @@ validate_maven_artifacts() {
local
jar_file="$REPO_DIR/java/target/paimon-vector-index-java-${RELEASE_VERSION}.jar"
local
sources_jar="$REPO_DIR/java/target/paimon-vector-index-java-${RELEASE_VERSION}-sources.jar"
local
javadoc_jar="$REPO_DIR/java/target/paimon-vector-index-java-${RELEASE_VERSION}-javadoc.jar"
+ local
ci_jar="$NATIVE_DIR/java-package/paimon-vector-index-java-${RELEASE_VERSION}.jar"
+ local
ci_sources_jar="$NATIVE_DIR/java-package/paimon-vector-index-java-${RELEASE_VERSION}-sources.jar"
+ local
ci_javadoc_jar="$NATIVE_DIR/java-package/paimon-vector-index-java-${RELEASE_VERSION}-javadoc.jar"
local artifact
local native_entry
-
- for artifact in "$jar_file" "$sources_jar" "$javadoc_jar"; do
+ local legal_entry
+
+ for artifact in \
+ "$jar_file" \
+ "$sources_jar" \
+ "$javadoc_jar" \
+ "$ci_jar" \
+ "$ci_sources_jar" \
+ "$ci_javadoc_jar"
+ do
if [[ ! -f "$artifact" ]]; then
echo "Expected Maven artifact is missing: $artifact" >&2
exit 1
fi
done
+ for artifact in "$sources_jar" "$ci_sources_jar"; do
+ if grep -Eq '^(native/|META-INF/LICENSE-binary$)' <<<"$(jar tf
"$artifact")"; then
+ echo "Sources jar contains binary-only resources: $artifact" >&2
+ exit 1
+ fi
+ done
+
for native_entry in \
native/linux/x86_64/libpaimon_vindex_jni.so \
native/linux/aarch64/libpaimon_vindex_jni.so \
@@ -512,6 +531,35 @@ validate_maven_artifacts() {
exit 1
fi
done
+
+ for legal_entry in \
+ META-INF/LICENSE \
+ META-INF/NOTICE \
+ META-INF/LICENSE-binary
+ do
+ if ! jar tf "$jar_file" | grep -qx "$legal_entry"; then
+ echo "Packaged jar is missing legal entry: $legal_entry" >&2
+ exit 1
+ fi
+ done
+
+ if ! jar tf "$jar_file" |
+ grep -qx 'org/apache/paimon/index/vector/NativeLibraryLoader.class'; then
+ echo "Packaged jar is missing NativeLibraryLoader.class" >&2
+ exit 1
+ fi
+
+ python3 "$REPO_DIR/tools/verify_binary_artifact.py" --jar "$jar_file"
"$ci_jar"
+
+ local test_classes="$REPO_DIR/java/target/test-classes"
+ if [[ ! -f
"$test_classes/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.class"
]]; then
+ echo "Packaged JAR loader smoke test class is missing from test output."
>&2
+ exit 1
+ fi
+ java -cp "$jar_file:$test_classes" \
+ org.apache.paimon.index.vector.VectorIndexNativeLoaderSmokeTest
+ java -cp "$ci_jar:$test_classes" \
+ org.apache.paimon.index.vector.VectorIndexNativeLoaderSmokeTest
}
if [[ "$DRY_RUN" == "true" ]]; then
diff --git a/tools/verify_binary_artifact.py b/tools/verify_binary_artifact.py
new file mode 100644
index 0000000..2310abb
--- /dev/null
+++ b/tools/verify_binary_artifact.py
@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Verify legal files and native resources in binary convenience artifacts."""
+
+import argparse
+import glob
+import json
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent.parent
+LEGAL_FILES = ("LICENSE", "NOTICE", "LICENSE-binary")
+WHEEL_LEGAL_SOURCES = {
+ "LICENSE": "LICENSE",
+ "NOTICE": "NOTICE",
+ "LICENSE-binary": "LICENSE-binary-ffi",
+}
+JAR_LEGAL_SOURCES = {
+ "LICENSE-binary": "LICENSE-binary",
+}
+JAR_NATIVE_FILES = (
+ "native/linux/x86_64/libpaimon_vindex_jni.so",
+ "native/linux/aarch64/libpaimon_vindex_jni.so",
+ "native/macos/aarch64/libpaimon_vindex_jni.dylib",
+ "native/windows/x86_64/paimon_vindex_jni.dll",
+)
+
+
+def resolve_artifacts(patterns):
+ artifacts = []
+ for pattern in patterns:
+ matches = sorted(glob.glob(pattern))
+ if not matches:
+ raise ValueError(f"artifact pattern matched no files: {pattern}")
+ artifacts.extend(Path(match) for match in matches)
+ return artifacts
+
+
+def require_entries(archive, artifact, entries):
+ names = set(archive.namelist())
+ missing = [entry for entry in entries if entry not in names]
+ if missing:
+ raise ValueError(f"{artifact} is missing entries: {',
'.join(missing)}")
+
+
+def require_canonical_content(archive, artifact, archive_prefix, sources):
+ for name, source in sources.items():
+ entry = f"{archive_prefix}/{name}"
+ expected = (ROOT / source).read_bytes()
+ actual = archive.read(entry)
+ if actual != expected:
+ raise ValueError(f"{artifact}!/{entry} does not match repository
{source}")
+
+
+def verify_wheel(artifact):
+ with zipfile.ZipFile(artifact) as archive:
+ names = archive.namelist()
+ dist_info_metadata = [
+ name for name in names if name.endswith(".dist-info/METADATA")
+ ]
+ if len(dist_info_metadata) != 1:
+ raise ValueError(
+ f"{artifact} must contain exactly one .dist-info/METADATA
entry"
+ )
+ dist_info = dist_info_metadata[0][: -len("METADATA")]
+ legal_entries = [f"{dist_info}licenses/{name}" for name in LEGAL_FILES]
+ package_legal_entries = [f"paimon_vindex/{name}" for name in
LEGAL_FILES]
+ require_entries(archive, artifact, legal_entries +
package_legal_entries)
+ require_canonical_content(
+ archive, artifact, "paimon_vindex", WHEEL_LEGAL_SOURCES
+ )
+ require_canonical_content(
+ archive,
+ artifact,
+ f"{dist_info}licenses".rstrip("/"),
+ WHEEL_LEGAL_SOURCES,
+ )
+
+ native_names = [
+ name
+ for name in names
+ if name.startswith("paimon_vindex/")
+ and name.endswith((".so", ".dylib", ".dll"))
+ ]
+ if len(native_names) != 1:
+ raise ValueError(
+ f"{artifact} must contain exactly one platform native library,
found "
+ f"{len(native_names)}"
+ )
+
+ print(f"Verified Python wheel: {artifact}")
+
+
+def verify_jar(artifact, native_files):
+ with zipfile.ZipFile(artifact) as archive:
+ required = (
+ "META-INF/LICENSE",
+ "META-INF/NOTICE",
+ "META-INF/LICENSE-binary",
+ "org/apache/paimon/index/vector/NativeLibraryLoader.class",
+ ) + tuple(native_files)
+ require_entries(archive, artifact, required)
+ require_canonical_content(
+ archive,
+ artifact,
+ "META-INF",
+ JAR_LEGAL_SOURCES,
+ )
+
+ print(f"Verified Java JAR: {artifact}")
+
+
+def verify_source_legal_inventory():
+ metadata = json.loads(
+ subprocess.check_output(
+ ["cargo", "metadata", "--format-version", "1", "--locked"],
+ cwd=ROOT,
+ text=True,
+ )
+ )
+ packages = {package["id"]: package for package in metadata["packages"]}
+ nodes = {node["id"]: node for node in metadata["resolve"]["nodes"]}
+ for root_name, license_name in (
+ ("paimon-vindex-ffi", "LICENSE-binary-ffi"),
+ ("paimon-vindex-jni", "LICENSE-binary"),
+ ):
+ root_id = next(
+ package["id"]
+ for package in metadata["packages"]
+ if package["source"] is None and package["name"] == root_name
+ )
+ resolved = set()
+ pending = [root_id]
+ while pending:
+ package_id = pending.pop()
+ if package_id in resolved:
+ continue
+ resolved.add(package_id)
+ for dependency in nodes[package_id]["deps"]:
+ if any(
+ kind["kind"] in (None, "normal")
+ for kind in dependency["dep_kinds"]
+ ):
+ pending.append(dependency["pkg"])
+
+ binary_license = (ROOT / license_name).read_text(encoding="utf-8")
+ missing = []
+ for package_id in resolved:
+ package = packages[package_id]
+ if package["source"] is None:
+ continue
+ expression = package.get("license") or ""
+ needs_additional_text = (
+ "Apache-2.0" not in expression or " AND " in expression
+ )
+ marker = f"{package['name']} {package['version']}"
+ if needs_additional_text and marker not in binary_license:
+ missing.append(f"{marker} ({expression})")
+
+ if missing:
+ raise ValueError(
+ f"{license_name} does not account for {root_name} runtime
dependencies: "
+ + ", ".join(sorted(missing))
+ )
+
+ for name in (
+ "LICENSE",
+ "NOTICE",
+ "LICENSE-binary",
+ "LICENSE-binary-ffi",
+ "ffi/DEPENDENCIES.rust.tsv",
+ "jni/DEPENDENCIES.rust.tsv",
+ ):
+ if not (ROOT / name).is_file():
+ raise ValueError(f"repository legal file is missing: {name}")
+
+ print("Verified binary legal inventory against native Rust dependencies")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--wheel", nargs="+", default=[], help="Wheel path or
glob")
+ parser.add_argument("--jar", nargs="+", default=[], help="JAR path or
glob")
+ parser.add_argument(
+ "--jar-native",
+ action="append",
+ choices=JAR_NATIVE_FILES,
+ help=(
+ "Native entry required in a JAR. Repeat for a partial-platform CI
build; "
+ "the default requires every release platform."
+ ),
+ )
+ parser.add_argument(
+ "--source",
+ action="store_true",
+ help="Verify LICENSE-binary against native Rust runtime dependencies",
+ )
+ args = parser.parse_args()
+
+ if not args.wheel and not args.jar and not args.source:
+ parser.error("at least one --source, --wheel, or --jar is required")
+
+ try:
+ if args.source:
+ verify_source_legal_inventory()
+ for artifact in resolve_artifacts(args.wheel):
+ verify_wheel(artifact)
+ for artifact in resolve_artifacts(args.jar):
+ verify_jar(artifact, args.jar_native or JAR_NATIVE_FILES)
+ except (OSError, ValueError, zipfile.BadZipFile, KeyError) as error:
+ print(f"Binary artifact verification failed: {error}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
