This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch 0.11.x
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/0.11.x by this push:
new 31586dba2 ci(python): bundle third-party licenses in pyiceberg-core
wheels (#3554)
31586dba2 is described below
commit 31586dba2868955c20092117254aeb3a3ddbc098
Author: Danny C Jones <[email protected]>
AuthorDate: Tue Oct 27 15:46:54 2026 +0000
ci(python): bundle third-party licenses in pyiceberg-core wheels (#3554)
Co-authored-by: Copilot App <[email protected]>
Co-authored-by: Kevin Liu <[email protected]>
Co-authored-by: Danny C Jones <[email protected]>
Co-authored-by: Kevin Liu <[email protected]>
---
.github/actions/generate-python-license/action.yml | 36 +++++++
.github/workflows/bindings_python_ci.yml | 1 +
.github/workflows/release_python.yml | 2 +
.github/workflows/release_python_nightly.yml | 3 +
.licenserc.yaml | 2 +
bindings/python/LICENSING.md | 72 ++++++++++++++
bindings/python/README.md | 4 +
bindings/python/about.hbs | 35 +++++++
bindings/python/about.toml | 108 +++++++++++++++++++++
bindings/python/third-party-licenses.txt | 27 ++++++
bindings/python/third-party-notices.txt | 42 ++++++++
website/src/release.md | 5 +
12 files changed, 337 insertions(+)
diff --git a/.github/actions/generate-python-license/action.yml
b/.github/actions/generate-python-license/action.yml
new file mode 100644
index 000000000..76b0e086d
--- /dev/null
+++ b/.github/actions/generate-python-license/action.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Generate wheel third-party licenses
+description: 'Regenerates bindings/python/LICENSE and NOTICE with the root
files plus the licenses and notices of the Rust dependencies bundled in the
wheels. The sdist keeps the root files.'
+runs:
+ using: "composite"
+ steps:
+ - name: Install cargo-about
+ uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 #
v2.87.21
+ with:
+ tool: [email protected]
+
+ - name: Generate LICENSE and NOTICE
+ shell: bash
+ working-directory: bindings/python
+ run: |
+ cargo about generate -c about.toml --locked --fail --manifest-path
Cargo.toml about.hbs > cargo-about-output.txt
+ # Both are symlinks to the root files, remove them so the redirects
don't write through.
+ rm LICENSE NOTICE
+ cat ../../LICENSE cargo-about-output.txt third-party-licenses.txt >
LICENSE
+ { cat ../../NOTICE; echo; cat third-party-notices.txt; } > NOTICE
diff --git a/.github/workflows/bindings_python_ci.yml
b/.github/workflows/bindings_python_ci.yml
index 6c9513199..eb77d8971 100644
--- a/.github/workflows/bindings_python_ci.yml
+++ b/.github/workflows/bindings_python_ci.yml
@@ -92,6 +92,7 @@ jobs:
with:
key: bindings-python
save-if: ${{ github.event_name == 'push' && github.ref ==
'refs/heads/main' }}
+ - uses: ./.github/actions/generate-python-license
- uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b #
v1.51.0
with:
working-directory: "bindings/python"
diff --git a/.github/workflows/release_python.yml
b/.github/workflows/release_python.yml
index 4dc866c56..301f1d614 100644
--- a/.github/workflows/release_python.yml
+++ b/.github/workflows/release_python.yml
@@ -142,6 +142,7 @@ jobs:
strategy:
max-parallel: 15
matrix:
+ # Keep `targets` in bindings/python/about.toml in sync with these.
include:
- { os: windows-latest }
- { os: macos-latest, target: "universal2-apple-darwin" }
@@ -183,6 +184,7 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ steps.get-msrv.outputs.msrv }}
+ - uses: ./.github/actions/generate-python-license
- uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b #
v1.51.0
with:
target: ${{ matrix.target }}
diff --git a/.github/workflows/release_python_nightly.yml
b/.github/workflows/release_python_nightly.yml
index dc10ebecf..600334812 100644
--- a/.github/workflows/release_python_nightly.yml
+++ b/.github/workflows/release_python_nightly.yml
@@ -75,6 +75,7 @@ jobs:
strategy:
max-parallel: 15
matrix:
+ # Keep `targets` in bindings/python/about.toml in sync with these.
include:
- { os: windows-latest }
- { os: macos-latest, target: "universal2-apple-darwin" }
@@ -116,6 +117,8 @@ jobs:
with:
rust-version: ${{ steps.get-msrv.outputs.msrv }}
+ - uses: ./.github/actions/generate-python-license
+
- uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b #
v1.51.0
with:
target: ${{ matrix.target }}
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 9959b0be4..6b48cf388 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -23,6 +23,8 @@ header:
paths-ignore:
- "LICENSE"
- "NOTICE"
+ - "bindings/python/third-party-licenses.txt"
+ - "bindings/python/third-party-notices.txt"
- ".gitattributes"
- "**/*.json"
# Generated content by mdbook
diff --git a/bindings/python/LICENSING.md b/bindings/python/LICENSING.md
new file mode 100644
index 000000000..7954046ac
--- /dev/null
+++ b/bindings/python/LICENSING.md
@@ -0,0 +1,72 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+# LICENSE and NOTICE for pyiceberg-core
+
+The `pyiceberg-core` wheels are a convenience artifact. We build them and
upload them to PyPI as part of every release.
+
+The Rust crates that `pyiceberg-core` depends on are statically linked into
each wheel. So the `LICENSE` and `NOTICE` inside a wheel must cover those
crates as well as iceberg-rust itself.
+
+The sdist only contains iceberg-rust's own source, so it ships the root
`LICENSE` and `NOTICE` unchanged.
+
+## How it works
+
+`LICENSE` and `NOTICE` are generated at build time, before `maturin` builds
and packages each wheel.
+
+The
[`generate-python-license`](../../.github/actions/generate-python-license/action.yml)
action regenerates the wheel's `LICENSE` and `NOTICE` in `bindings/python/`:
+
+- `LICENSE`: the root `LICENSE`, then the
[`cargo-about`](https://github.com/EmbarkStudios/cargo-about) output, then the
manual entries in `third-party-licenses.txt`.
+- `NOTICE`: the root `NOTICE`, then the manual entries in
`third-party-notices.txt`.
+
+The action is used in these workflows:
+
+- [`release_python.yml`](../../.github/workflows/release_python.yml): the
release wheels published to PyPI.
+-
[`release_python_nightly.yml`](../../.github/workflows/release_python_nightly.yml):
the nightly wheels published to TestPyPI.
+- [`bindings_python_ci.yml`](../../.github/workflows/bindings_python_ci.yml):
pull requests. It only checks that generation doesn't fail. The generated files
are not published.
+
+Note that in the repository, `bindings/python/LICENSE` and `NOTICE` are
symlinks to the root files, so builds outside the release workflows ship the
plain root files.
+
+## Relevant files
+
+- `about.hbs`: the `cargo-about` output template. Keep it a plain template.
+- `about.toml`: configures `cargo-about`.
+ - `accepted`: the licenses we bundle, matching `deny.toml`. Add new ones at
the end, as the order is the preference when a crate offers a choice.
+ - `targets`: the wheel targets. Keep in sync with the build matrices in
`release_python.yml` and `release_python_nightly.yml`.
+ - Clarifications, for crates whose license files `cargo-about` can't read
(such as `libm`). They pin the file by checksum (`shasum -a 256 <file>`). See
the [`cargo-about`
docs](https://embarkstudios.github.io/cargo-about/cli/clarify.html).
+- `third-party-licenses.txt`: otherwise, add the license text here. For
example, `brotli-decompressor`'s MIT license, which the crate doesn't ship. It
names the crate and version, so re-check it when the crate is upgraded.
+- `third-party-notices.txt`: the `NOTICE` entries of dependencies that ship
one (such as Arrow, Avro, DataFusion, OpenDAL and moka). Find them from
`bindings/python` (the repository's own crates appear too):
+
+ ```shell
+ cargo about generate -c about.toml --locked --fail --manifest-path
Cargo.toml --format json \
+ | jq -r '.crates[].package.manifest_path' | xargs -n1 dirname \
+ | xargs -I{} find {} -maxdepth 1 -iname 'NOTICE*'
+ ```
+
+## Keeping entries up to date
+
+Pull request CI runs the action, so it fails when generation fails, for
example on a license that isn't in `accepted`.
+
+Everything else needs a manual check and update before every release: the
clarifications and `targets` in `about.toml`, `third-party-licenses.txt` and
`third-party-notices.txt`. They don't change with the dependencies.
+
+Run this from `bindings/python` to check. A `WARN` or `warning:` on stderr
means a clarification no longer applies, and `<year>` in the output means no
license file was matched:
+
+```shell
+cargo install cargo-about --version 0.9.2 --locked --features cli
+cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml
about.hbs | grep -nE '<year>|<owner>|<copyright holders>'
+```
diff --git a/bindings/python/README.md b/bindings/python/README.md
index 320a9a401..f831c28e8 100644
--- a/bindings/python/README.md
+++ b/bindings/python/README.md
@@ -46,3 +46,7 @@ make build
```shell
make test
```
+
+## LICENSE and NOTICE
+
+The wheels bundle the licenses and notices of their Rust dependencies. See
[LICENSING.md](LICENSING.md) for how they are generated and how to keep them up
to date.
diff --git a/bindings/python/about.hbs b/bindings/python/about.hbs
new file mode 100644
index 000000000..c55b318a1
--- /dev/null
+++ b/bindings/python/about.hbs
@@ -0,0 +1,35 @@
+{{!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+--}}
+
+--------------------------------------------------------------------------------
+
+This package bundles Rust dependencies. Their licenses follow.
+{{#each licenses}}
+
+--------------------------------------------------------------------------------
+
+{{{name}}}
+
+Used by:
+{{#each used_by}}
+ - {{crate.name}} {{crate.version}}{{#if crate.repository}}
({{crate.repository}}){{/if}}
+{{/each}}
+
+{{{text}}}
+{{/each}}
diff --git a/bindings/python/about.toml b/bindings/python/about.toml
new file mode 100644
index 000000000..a41117000
--- /dev/null
+++ b/bindings/python/about.toml
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Configuration for `cargo about`, which lists the license texts of the Rust
+# dependencies compiled into pyiceberg-core.
+#
+# Keep `accepted` in line with deny.toml. Order is preference when a crate
offers
+# a choice, so add new licenses at the end.
+#
+# Accepted licenses must be ASF Category A or B:
+# https://www.apache.org/legal/resolved.html
+accepted = [
+ "Apache-2.0",
+ "BSD-3-Clause",
+ "CC0-1.0",
+ "ISC",
+ "MIT",
+ # Not on that page; Category A per
+ # https://issues.apache.org/jira/browse/LEGAL-660
+ "Unicode-3.0",
+ "Zlib",
+ "Apache-2.0 WITH LLVM-exception",
+ "BSD-2-Clause",
+ "BSL-1.0",
+ "CDLA-Permissive-2.0",
+ "bzip2-1.0.6",
+]
+ignore-build-dependencies = true
+ignore-dev-dependencies = true
+# The wheel targets in release_python.yml and release_python_nightly.yml. Keep
in sync.
+# 0.11.x branch builds no windows-arm wheel, so aarch64-pc-windows-msvc is
absent.
+targets = [
+ "aarch64-apple-darwin",
+ "aarch64-unknown-linux-gnu",
+ "armv7-unknown-linux-gnueabihf",
+ "x86_64-apple-darwin",
+ "x86_64-pc-windows-msvc",
+ "x86_64-unknown-linux-gnu",
+]
+
+# The published crate omits its LICENSE, so cargo-about fetches it from the
+# upstream repository.
+[alloc-stdlib.clarify]
+license = "BSD-3-Clause"
+[[alloc-stdlib.clarify.git]]
+checksum = "c0c56f26d9c051cac4d200c34c84e7ae9aaa853e01a982a1df08b09931e518ae"
+path = "LICENSE"
+
+# libm's license is in LICENSE.txt, which cargo-about doesn't match by name.
+[libm.clarify]
+license = "MIT"
+[[libm.clarify.files]]
+checksum = "3823dda7cf046602f4b4e77ec8e227863dc4736037cc85bb33d9f19febe16bb7"
+path = "LICENSE.txt"
+
+# zstd-sys vendors zstd's C code (BSD-3-Clause, Meta) next to the Rust
+# bindings (BSD-3-Clause, Facebook); neither is in its Cargo.toml license.
+[zstd-sys.clarify]
+license = "MIT AND BSD-3-Clause"
+[[zstd-sys.clarify.files]]
+checksum = "129e8edef29e9abcd2ebabe252f4ef1b1289cdca356bf0040284a2fbccfb96c8"
+license = "MIT"
+path = "LICENSE.Mit"
+[[zstd-sys.clarify.files]]
+checksum = "48341f685c87304089aa099b23c386f8bacc519ef555aa7a13e239908907b3fd"
+license = "BSD-3-Clause"
+path = "LICENSE.BSD-3-Clause"
+[[zstd-sys.clarify.files]]
+checksum = "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8"
+license = "BSD-3-Clause"
+path = "zstd/LICENSE"
+
+# aws-lc-sys and aws-lc-rs each ship all of their licenses in one LICENSE file,
+# which cargo-about doesn't match. Point every license at that file, at the
cost
+# of repeating its text.
+[aws-lc-sys.clarify]
+license = "Apache-2.0 AND ISC AND MIT AND BSD-3-Clause"
+[[aws-lc-sys.clarify.files]]
+checksum = "728536b4160e051f86d7c9c388f704866b3d512cd7df97ac3516c65279523c4e"
+path = "LICENSE"
+
+[aws-lc-rs.clarify]
+license = "Apache-2.0 AND ISC"
+[[aws-lc-rs.clarify.files]]
+checksum = "b50b376e7d24a0598488b730c3034ffcd0b58dd36c0913a24b9903a3cfd04bf7"
+path = "LICENSE"
+
+# For individual crate exceptions, refer to deny.toml.
+
+[colored]
+accepted = ["MPL-2.0"]
+
+[option-ext]
+accepted = ["MPL-2.0"]
diff --git a/bindings/python/third-party-licenses.txt
b/bindings/python/third-party-licenses.txt
new file mode 100644
index 000000000..b19fd73c5
--- /dev/null
+++ b/bindings/python/third-party-licenses.txt
@@ -0,0 +1,27 @@
+--------------------------------------------------------------------------------
+
+MIT License
+
+Used by:
+ - brotli-decompressor 5.0.3
(https://github.com/dropbox/rust-brotli-decompressor)
+ (src/context.rs)
+
+Copyright 2013 Google Inc. All Rights Reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/bindings/python/third-party-notices.txt
b/bindings/python/third-party-notices.txt
new file mode 100644
index 000000000..0ba350c2f
--- /dev/null
+++ b/bindings/python/third-party-notices.txt
@@ -0,0 +1,42 @@
+This package bundles the following Rust dependencies that ship their own
NOTICE.
+
+Apache Arrow (arrow, parquet and related crates)
+Copyright 2016-2026 The Apache Software Foundation
+
+This product includes software from the chronoutil crate (MIT)
+ * Copyright (c) 2020-2022 Oliver Margetts
+ * https://github.com/olliemath/chronoutil
+
+This product includes software from the compact-thrift project (Apache 2.0)
+ * Copyright Jörn Horstmann
+ * https://github.com/jhorstmann/compact-thrift
+
+Apache Avro (apache-avro)
+Copyright 2010-2025 The Apache Software Foundation
+
+The initial implementation of the Apache Avro Rust SDK has been
+developed by Flavien Raynaud (https://github.com/flavray/avro-rs)
+and donated to the Apache Avro project in 2020.
+
+Apache Arrow Object Store (object_store)
+Copyright 2020-2024 The Apache Software Foundation
+
+Apache DataFusion (datafusion and related crates)
+Copyright 2019-2026 The Apache Software Foundation
+
+Apache OpenDAL (opendal and related crates)
+Copyright 2023-2026 The Apache Software Foundation
+
+Apache OpenDAL Reqsign (reqsign-core and related crates)
+Copyright 2025-2026 The Apache Software Foundation
+
+Moka (moka)
+
+The majority of the Moka library is dual-licensed under the MIT and Apache 2.0
+licenses. However, the following files are licensed solely under the Apache
+License 2.0:
+
+- src/common/frequency_sketch.rs
+- src/common/timer_wheel.rs
+
+These files were ported from the Java Caffeine library and are not
dual-licensed.
diff --git a/website/src/release.md b/website/src/release.md
index 4b97fc6f7..c9812ecb3 100644
--- a/website/src/release.md
+++ b/website/src/release.md
@@ -174,6 +174,11 @@ Run the following command to verify the licenses meet the
project's policy.
dev/release/dependencies.sh check
```
+#### Check pyiceberg-core LICENSE and NOTICE
+
+The `pyiceberg-core` wheels bundle third-party licenses and notices, and some
of the entries are maintained by hand.
+Check and update them following
[`bindings/python/LICENSING.md`](https://github.com/apache/iceberg-rust/blob/main/bindings/python/LICENSING.md#keeping-entries-up-to-date).
+
#### Open pull request
Open a pull request with all the changes.
