This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3493-26356f549b24b792e245d6e91240ebaa9f073c88 in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit 32acec67ab7460e408e313c0b9c2abf9482d2b86 Author: Kevin Liu <[email protected]> AuthorDate: Mon Oct 26 17:24:08 2026 +0000 ci(python): bundle third-party licenses in pyiceberg-core wheels (#3493) * ci(python): bundle third-party licenses in pyiceberg-core wheels Co-authored-by: Copilot App <[email protected]> * build(python): add third-party notices to pyiceberg-core NOTICE Co-authored-by: Copilot App <[email protected]> * build(python): add zstd-sys license clarification Co-authored-by: Copilot App <[email protected]> * build(python): add brotli-decompressor MIT license to LICENSE Co-authored-by: Copilot App <[email protected]> * build(python): add libm license clarification Co-authored-by: Copilot App <[email protected]> * build(python): add alloc-stdlib license clarification Co-authored-by: Copilot App <[email protected]> * build(python): limit third-party licenses to the wheel targets Co-authored-by: Copilot App <[email protected]> * docs(python): document the LICENSE and NOTICE flow Co-authored-by: Copilot App <[email protected]> * docs(release): add a pyiceberg-core LICENSE and NOTICE check Co-authored-by: Copilot App <[email protected]> --------- Co-authored-by: Copilot App <[email protected]> --- .github/actions/generate-python-license/action.yml | 36 ++++++++++ .github/workflows/bindings_python_ci.yml | 1 + .github/workflows/release_python.yml | 2 + .github/workflows/release_python_nightly.yml | 3 + .licenserc.yaml | 2 + bindings/python/LICENSING.md | 72 +++++++++++++++++++ bindings/python/README.md | 4 ++ bindings/python/about.hbs | 35 +++++++++ bindings/python/about.toml | 84 ++++++++++++++++++++++ bindings/python/third-party-licenses.txt | 27 +++++++ bindings/python/third-party-notices.txt | 30 ++++++++ website/src/release.md | 5 ++ 12 files changed, 301 insertions(+) diff --git a/.github/actions/generate-python-license/action.yml b/.github/actions/generate-python-license/action.yml new file mode 100644 index 000000000..76b0e086d --- /dev/null +++ b/.github/actions/generate-python-license/action.yml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Generate wheel third-party licenses +description: 'Regenerates bindings/python/LICENSE and NOTICE with the root files plus the licenses and notices of the Rust dependencies bundled in the wheels. The sdist keeps the root files.' +runs: + using: "composite" + steps: + - name: Install cargo-about + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 + with: + tool: [email protected] + + - name: Generate LICENSE and NOTICE + shell: bash + working-directory: bindings/python + run: | + cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml about.hbs > cargo-about-output.txt + # Both are symlinks to the root files, remove them so the redirects don't write through. + rm LICENSE NOTICE + cat ../../LICENSE cargo-about-output.txt third-party-licenses.txt > LICENSE + { cat ../../NOTICE; echo; cat third-party-notices.txt; } > NOTICE diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 6fc15f11e..a41d5bf12 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -93,6 +93,7 @@ jobs: with: key: bindings-python save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + - uses: $/.github/actions/generate-python-license - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: working-directory: "bindings/python" diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index b5632dfc7..79758535c 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -142,6 +142,7 @@ jobs: strategy: max-parallel: 15 matrix: + # Keep `targets` in bindings/python/about.toml in sync with these. include: - { os: windows-latest } - { os: windows-11-arm } @@ -184,6 +185,7 @@ jobs: uses: $/.github/actions/setup-builder with: rust-version: ${{ steps.get-msrv.outputs.msrv }} + - uses: $/.github/actions/generate-python-license - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.target }} diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index 68ba70c79..c6deddc31 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -79,6 +79,7 @@ jobs: strategy: max-parallel: 15 matrix: + # Keep `targets` in bindings/python/about.toml in sync with these. include: - { os: windows-latest } - { os: windows-11-arm } @@ -121,6 +122,8 @@ jobs: with: rust-version: ${{ steps.get-msrv.outputs.msrv }} + - uses: $/.github/actions/generate-python-license + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.target }} diff --git a/.licenserc.yaml b/.licenserc.yaml index 5b3ca98e9..a3b586dff 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -23,6 +23,8 @@ header: paths-ignore: - "LICENSE" - "NOTICE" + - "bindings/python/third-party-licenses.txt" + - "bindings/python/third-party-notices.txt" - ".gitattributes" - "**/*.json" # Generated content by mdbook diff --git a/bindings/python/LICENSING.md b/bindings/python/LICENSING.md new file mode 100644 index 000000000..6a8370ae5 --- /dev/null +++ b/bindings/python/LICENSING.md @@ -0,0 +1,72 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. +--> + +# LICENSE and NOTICE for pyiceberg-core + +The `pyiceberg-core` wheels are a convenience artifact. We build them and upload them to PyPI as part of every release. + +The Rust crates that `pyiceberg-core` depends on are statically linked into each wheel. So the `LICENSE` and `NOTICE` inside a wheel must cover those crates as well as iceberg-rust itself. + +The sdist only contains iceberg-rust's own source, so it ships the root `LICENSE` and `NOTICE` unchanged. + +## How it works + +`LICENSE` and `NOTICE` are generated at build time, before `maturin` builds and packages each wheel. + +The [`generate-python-license`](../../.github/actions/generate-python-license/action.yml) action regenerates the wheel's `LICENSE` and `NOTICE` in `bindings/python/`: + +- `LICENSE`: the root `LICENSE`, then the [`cargo-about`](https://github.com/EmbarkStudios/cargo-about) output, then the manual entries in `third-party-licenses.txt`. +- `NOTICE`: the root `NOTICE`, then the manual entries in `third-party-notices.txt`. + +The action is used in these workflows: + +- [`release_python.yml`](../../.github/workflows/release_python.yml): the release wheels published to PyPI. +- [`release_python_nightly.yml`](../../.github/workflows/release_python_nightly.yml): the nightly wheels published to TestPyPI. +- [`bindings_python_ci.yml`](../../.github/workflows/bindings_python_ci.yml): pull requests. It only checks that generation doesn't fail. The generated files are not published. + +Note that in the repository, `bindings/python/LICENSE` and `NOTICE` are symlinks to the root files, so builds outside the release workflows ship the plain root files. + +## Relevant files + +- `about.hbs`: the `cargo-about` output template. Keep it a plain template. +- `about.toml`: configures `cargo-about`. + - `accepted`: the licenses we bundle, matching `deny.toml`. Add new ones at the end, as the order is the preference when a crate offers a choice. + - `targets`: the wheel targets. Keep in sync with the build matrices in `release_python.yml` and `release_python_nightly.yml`. + - Clarifications, for crates whose license files `cargo-about` can't read (such as `libm`). They pin the file by checksum (`shasum -a 256 <file>`). See the [`cargo-about` docs](https://embarkstudios.github.io/cargo-about/cli/clarify.html). +- `third-party-licenses.txt`: otherwise, add the license text here. For example, `brotli-decompressor`'s MIT license, which the crate doesn't ship. It names the crate and version, so re-check it when the crate is upgraded. +- `third-party-notices.txt`: the `NOTICE` entries of dependencies that ship one (Arrow, Avro, moka). Find them from `bindings/python` (the repository's own crates appear too): + + ```shell + cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml --format json \ + | jq -r '.crates[].package.manifest_path' | xargs -n1 dirname \ + | xargs -I{} find {} -maxdepth 1 -iname 'NOTICE*' + ``` + +## Keeping entries up to date + +Pull request CI runs the action, so it fails when generation fails, for example on a license that isn't in `accepted`. + +Everything else needs a manual check and update before every release: the clarifications and `targets` in `about.toml`, `third-party-licenses.txt` and `third-party-notices.txt`. They don't change with the dependencies. + +Run this from `bindings/python` to check. A `WARN` or `warning:` on stderr means a clarification no longer applies, and `<year>` in the output means no license file was matched: + +```shell +cargo install cargo-about --version 0.9.2 --locked --features cli +cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml about.hbs | grep -nE '<year>|<owner>|<copyright holders>' +``` diff --git a/bindings/python/README.md b/bindings/python/README.md index 320a9a401..f831c28e8 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -46,3 +46,7 @@ make build ```shell make test ``` + +## LICENSE and NOTICE + +The wheels bundle the licenses and notices of their Rust dependencies. See [LICENSING.md](LICENSING.md) for how they are generated and how to keep them up to date. diff --git a/bindings/python/about.hbs b/bindings/python/about.hbs new file mode 100644 index 000000000..c55b318a1 --- /dev/null +++ b/bindings/python/about.hbs @@ -0,0 +1,35 @@ +{{!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--}} + +-------------------------------------------------------------------------------- + +This package bundles Rust dependencies. Their licenses follow. +{{#each licenses}} + +-------------------------------------------------------------------------------- + +{{{name}}} + +Used by: +{{#each used_by}} + - {{crate.name}} {{crate.version}}{{#if crate.repository}} ({{crate.repository}}){{/if}} +{{/each}} + +{{{text}}} +{{/each}} diff --git a/bindings/python/about.toml b/bindings/python/about.toml new file mode 100644 index 000000000..2a1a0fd2e --- /dev/null +++ b/bindings/python/about.toml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Configuration for `cargo about`, which lists the license texts of the Rust +# dependencies compiled into pyiceberg-core. +# +# Keep `accepted` in line with deny.toml. Order is preference when a crate offers +# a choice, so add new licenses at the end. +# +# Accepted licenses must be ASF Category A or B: +# https://www.apache.org/legal/resolved.html +accepted = [ + "Apache-2.0", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "MIT", + # Not on that page; Category A per + # https://issues.apache.org/jira/browse/LEGAL-660 + "Unicode-3.0", + "Zlib", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSL-1.0", + "CDLA-Permissive-2.0", +] +ignore-build-dependencies = true +ignore-dev-dependencies = true +# The wheel targets in release_python.yml and release_python_nightly.yml. Keep in sync. +targets = [ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "armv7-unknown-linux-gnueabihf", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", +] + +# The published crate omits its LICENSE, so cargo-about fetches it from the +# upstream repository. +[alloc-stdlib.clarify] +license = "BSD-3-Clause" +[[alloc-stdlib.clarify.git]] +checksum = "c0c56f26d9c051cac4d200c34c84e7ae9aaa853e01a982a1df08b09931e518ae" +path = "LICENSE" + +# libm's license is in LICENSE.txt, which cargo-about doesn't match by name. +[libm.clarify] +license = "MIT" +[[libm.clarify.files]] +checksum = "3823dda7cf046602f4b4e77ec8e227863dc4736037cc85bb33d9f19febe16bb7" +path = "LICENSE.txt" + +# zstd-sys vendors zstd's C code (BSD-3-Clause, Meta) next to the Rust +# bindings (BSD-3-Clause, Facebook); neither is in its Cargo.toml license. +[zstd-sys.clarify] +license = "MIT AND BSD-3-Clause" +[[zstd-sys.clarify.files]] +checksum = "129e8edef29e9abcd2ebabe252f4ef1b1289cdca356bf0040284a2fbccfb96c8" +license = "MIT" +path = "LICENSE.Mit" +[[zstd-sys.clarify.files]] +checksum = "48341f685c87304089aa099b23c386f8bacc519ef555aa7a13e239908907b3fd" +license = "BSD-3-Clause" +path = "LICENSE.BSD-3-Clause" +[[zstd-sys.clarify.files]] +checksum = "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" +license = "BSD-3-Clause" +path = "zstd/LICENSE" diff --git a/bindings/python/third-party-licenses.txt b/bindings/python/third-party-licenses.txt new file mode 100644 index 000000000..b19fd73c5 --- /dev/null +++ b/bindings/python/third-party-licenses.txt @@ -0,0 +1,27 @@ +-------------------------------------------------------------------------------- + +MIT License + +Used by: + - brotli-decompressor 5.0.3 (https://github.com/dropbox/rust-brotli-decompressor) + (src/context.rs) + +Copyright 2013 Google Inc. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/python/third-party-notices.txt b/bindings/python/third-party-notices.txt new file mode 100644 index 000000000..74fef989e --- /dev/null +++ b/bindings/python/third-party-notices.txt @@ -0,0 +1,30 @@ +This package bundles the following Rust dependencies that ship their own NOTICE. + +Apache Arrow (arrow, parquet and related crates) +Copyright 2016-2026 The Apache Software Foundation + +This product includes software from the chronoutil crate (MIT) + * Copyright (c) 2020-2022 Oliver Margetts + * https://github.com/olliemath/chronoutil + +This product includes software from the compact-thrift project (Apache 2.0) + * Copyright Jörn Horstmann + * https://github.com/jhorstmann/compact-thrift + +Apache Avro (apache-avro) +Copyright 2010-2025 The Apache Software Foundation + +The initial implementation of the Apache Avro Rust SDK has been +developed by Flavien Raynaud (https://github.com/flavray/avro-rs) +and donated to the Apache Avro project in 2020. + +Moka (moka) + +The majority of the Moka library is dual-licensed under the MIT and Apache 2.0 +licenses. However, the following files are licensed solely under the Apache +License 2.0: + +- src/common/frequency_sketch.rs +- src/common/timer_wheel.rs + +These files were ported from the Java Caffeine library and are not dual-licensed. diff --git a/website/src/release.md b/website/src/release.md index a519fce56..5a7aad146 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -178,6 +178,11 @@ Run the following command to verify the licenses meet the project's policy. python3 dev/release/licenses.py dependencies check ``` +#### Check pyiceberg-core LICENSE and NOTICE + +The `pyiceberg-core` wheels bundle third-party licenses and notices, and some of the entries are maintained by hand. +Check and update them following [`bindings/python/LICENSING.md`](https://github.com/apache/iceberg-rust/blob/main/bindings/python/LICENSING.md#keeping-entries-up-to-date). + #### Open pull request Open a pull request with all the changes.
