This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-mosaic.git
The following commit(s) were added to refs/heads/main by this push:
new 178fdd9 Add release tooling scripts and unify CI infrastructure (#27)
178fdd9 is described below
commit 178fdd9261d81de260987f34b31b35866bebdeae
Author: Jingsong Lee <[email protected]>
AuthorDate: Thu May 21 14:20:44 2026 +0800
Add release tooling scripts and unify CI infrastructure (#27)
- Flatten tools/releasing/ into tools/ for simpler layout
- Add tools/validate_asf_yaml.py for .asf.yaml schema validation
- Add tools/dependencies.py for cargo-deny license check and
DEPENDENCIES.rust.tsv generation
- Upgrade GitHub Actions versions to latest (checkout@v6, cache@v5,
artifact@v5)
- Add .asf.yaml validation step to CI check job
- Update docs/creating-a-release.html with new script paths and dependency
license step
---
.github/workflows/ci.yml | 3 +
.github/workflows/publish_snapshot.yml | 18 +--
.github/workflows/release-java.yml | 14 +--
.github/workflows/release-python.yml | 14 +--
docs/creating-a-release.html | 24 +++-
tools/{releasing => }/create_release_branch.sh | 0
tools/{releasing => }/create_source_release.sh | 2 +-
tools/dependencies.py | 124 ++++++++++++++++++++
tools/{releasing => }/update_branch_version.sh | 0
tools/validate_asf_yaml.py | 153 +++++++++++++++++++++++++
10 files changed, 324 insertions(+), 28 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5f81ac2..fc55cf1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -36,6 +36,9 @@ jobs:
steps:
- uses: actions/checkout@v6
+ - name: Validate .asf.yaml
+ run: pip install pyyaml -q && python3 tools/validate_asf_yaml.py
+
- name: Format
run: cargo fmt --all -- --check
diff --git a/.github/workflows/publish_snapshot.yml
b/.github/workflows/publish_snapshot.yml
index cb2c5ad..0d27d8b 100644
--- a/.github/workflows/publish_snapshot.yml
+++ b/.github/workflows/publish_snapshot.yml
@@ -59,7 +59,7 @@ jobs:
arch: x86_64
lib_name: paimon_mosaic_jni.dll
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Setup Rust toolchain
run: |
@@ -67,7 +67,7 @@ jobs:
rustup default stable
- name: Cache Rust dependencies
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: |
~/.cargo/registry
@@ -81,7 +81,7 @@ jobs:
run: cargo build --release -p paimon-mosaic-jni
- name: Upload native library
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v5
with:
name: native-${{ matrix.os_name }}-${{ matrix.arch }}
path: target/release/${{ matrix.lib_name }}
@@ -91,28 +91,28 @@ jobs:
runs-on: ubuntu-latest
needs: [build-native]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Download linux x86_64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-linux-x86_64
path: java/src/main/resources/native/linux/x86_64
- name: Download linux aarch64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-linux-aarch64
path: java/src/main/resources/native/linux/aarch64
- name: Download macOS aarch64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-macos-aarch64
path: java/src/main/resources/native/macos/aarch64
- name: Download windows x86_64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-windows-x86_64
path: java/src/main/resources/native/windows/x86_64
@@ -127,7 +127,7 @@ jobs:
distribution: 'temurin'
- name: Cache local Maven repository
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: ~/.m2/repository
key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
diff --git a/.github/workflows/release-java.yml
b/.github/workflows/release-java.yml
index 423f368..519b5b2 100644
--- a/.github/workflows/release-java.yml
+++ b/.github/workflows/release-java.yml
@@ -60,7 +60,7 @@ jobs:
arch: x86_64
lib_name: paimon_mosaic_jni.dll
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Setup Rust toolchain
run: |
@@ -71,7 +71,7 @@ jobs:
run: cargo build --release -p paimon-mosaic-jni
- name: Upload native library
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v5
with:
name: native-${{ matrix.os_name }}-${{ matrix.arch }}
path: target/release/${{ matrix.lib_name }}
@@ -81,28 +81,28 @@ jobs:
runs-on: ubuntu-latest
needs: [build-native]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Download linux x86_64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-linux-x86_64
path: java/src/main/resources/native/linux/x86_64
- name: Download linux aarch64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-linux-aarch64
path: java/src/main/resources/native/linux/aarch64
- name: Download macOS aarch64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-macos-aarch64
path: java/src/main/resources/native/macos/aarch64
- name: Download windows x86_64 native library
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: native-windows-x86_64
path: java/src/main/resources/native/windows/x86_64
diff --git a/.github/workflows/release-python.yml
b/.github/workflows/release-python.yml
index 4c24155..38de40a 100644
--- a/.github/workflows/release-python.yml
+++ b/.github/workflows/release-python.yml
@@ -50,7 +50,7 @@ jobs:
- os: ubuntu-24.04-arm
arch: aarch64
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Inject RC version into pyproject.toml
if: contains(github.ref, '-rc')
@@ -81,7 +81,7 @@ jobs:
cp target/release/libpaimon_mosaic_ffi.so {package}/mosaic/
- name: Upload wheels
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v5
with:
name: wheels-linux-${{ matrix.arch }}
path: wheelhouse/*.whl
@@ -96,7 +96,7 @@ jobs:
target: aarch64-apple-darwin
lib_name: libpaimon_mosaic_ffi.dylib
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Inject RC version into pyproject.toml
if: contains(github.ref, '-rc')
@@ -135,7 +135,7 @@ jobs:
mv python/dist/repaired/*.whl python/dist/
- name: Upload wheel
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v5
with:
name: wheels-${{ matrix.os }}-${{ matrix.target }}
path: python/dist/*.whl
@@ -143,7 +143,7 @@ jobs:
wheels-windows:
runs-on: windows-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Inject RC version into pyproject.toml
if: contains(github.ref, '-rc')
@@ -178,7 +178,7 @@ jobs:
run: python -m build --wheel
- name: Upload wheel
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v5
with:
name: wheels-windows-x86_64
path: python/dist/*.whl
@@ -191,7 +191,7 @@ jobs:
needs: [wheels-linux, wheels-macos, wheels-windows]
if: startsWith(github.ref, 'refs/tags/')
steps:
- - uses: actions/download-artifact@v4
+ - uses: actions/download-artifact@v5
with:
pattern: wheels-*
merge-multiple: true
diff --git a/docs/creating-a-release.html b/docs/creating-a-release.html
index c3d06c7..39cfbf9 100644
--- a/docs/creating-a-release.html
+++ b/docs/creating-a-release.html
@@ -159,14 +159,14 @@ RC_NUM="1"</code></pre>
<h3>Create a Release Branch</h3>
<pre><code>cd tools
-RELEASE_VERSION=${RELEASE_VERSION} RELEASE_CANDIDATE=${RC_NUM}
./releasing/create_release_branch.sh</code></pre>
+RELEASE_VERSION=${RELEASE_VERSION} RELEASE_CANDIDATE=${RC_NUM}
./create_release_branch.sh</code></pre>
<p>This creates a branch named
<code>release-${RELEASE_VERSION}-rc${RC_NUM}</code> from the current HEAD.</p>
<h3>Bump Version on Main</h3>
<p>After cutting the release branch, bump <code>main</code> to the
next development version:</p>
<pre><code>git checkout main
cd tools
-OLD_VERSION=${RELEASE_VERSION} NEW_VERSION=${NEXT_VERSION}-SNAPSHOT
./releasing/update_branch_version.sh</code></pre>
+OLD_VERSION=${RELEASE_VERSION} NEW_VERSION=${NEXT_VERSION}-SNAPSHOT
./update_branch_version.sh</code></pre>
<p>The script updates version strings in all <code>pom.xml</code>
(matching both <code>${OLD_VERSION}</code> and
<code>${OLD_VERSION}-SNAPSHOT</code>), <code>Cargo.toml</code> (excluding
<code>target/</code>), and <code>python/pyproject.toml</code> files, then
creates a commit.</p>
<!-- ============================================================
-->
@@ -176,7 +176,7 @@ OLD_VERSION=${RELEASE_VERSION}
NEW_VERSION=${NEXT_VERSION}-SNAPSHOT ./releasing/
<p>Before tagging, update the release branch to non-SNAPSHOT
versions (removing <code>-SNAPSHOT</code> from <code>pom.xml</code>):</p>
<pre><code>git checkout release-${RELEASE_VERSION}-rc${RC_NUM}
cd tools
-OLD_VERSION=${RELEASE_VERSION}-SNAPSHOT NEW_VERSION=${RELEASE_VERSION}
./releasing/update_branch_version.sh
+OLD_VERSION=${RELEASE_VERSION}-SNAPSHOT NEW_VERSION=${RELEASE_VERSION}
./update_branch_version.sh
cd ..</code></pre>
<p>This ensures the Java artifacts deployed from the RC tag carry
the final release version (e.g. <code>0.1.0</code>) rather than a SNAPSHOT
suffix.</p>
@@ -192,9 +192,25 @@ git push origin
v${RELEASE_VERSION}-rc${RC_NUM}</code></pre>
<li><strong>Release Python</strong> — builds wheels for
4 platforms, publishes to TestPyPI</li>
</ul>
+ <h3>Verify and Generate Dependency Licenses</h3>
+ <p>ASF releases must include a declaration of third-party
dependency licenses. Use <code>cargo-deny</code> to check and generate the
license manifest:</p>
+<pre><code># Install cargo-deny (one-time)
+cargo install cargo-deny
+
+# Check all dependency licenses are approved
+python3 tools/dependencies.py check
+
+# Generate DEPENDENCIES.rust.tsv
+python3 tools/dependencies.py generate
+
+# Commit the generated files (required for inclusion in the source tarball)
+git add DEPENDENCIES.rust.tsv core/DEPENDENCIES.rust.tsv
ffi/DEPENDENCIES.rust.tsv jni/DEPENDENCIES.rust.tsv
+git commit -m "Generate DEPENDENCIES.rust.tsv for release
${RELEASE_VERSION}"</code></pre>
+ <p>Fix any license violations before proceeding. The generated
files must be committed because <code>create_source_release.sh</code> builds
the archive from a git clone.</p>
+
<h3>Create Source Release Artifacts</h3>
<pre><code>cd tools
-RELEASE_VERSION=${RELEASE_VERSION}
./releasing/create_source_release.sh</code></pre>
+RELEASE_VERSION=${RELEASE_VERSION} ./create_source_release.sh</code></pre>
<p>This creates the following under
<code>tools/release/</code>:</p>
<ul>
<li><code>apache-paimon-mosaic-${RELEASE_VERSION}-src.tgz</code> — source
archive</li>
diff --git a/tools/releasing/create_release_branch.sh
b/tools/create_release_branch.sh
similarity index 100%
rename from tools/releasing/create_release_branch.sh
rename to tools/create_release_branch.sh
diff --git a/tools/releasing/create_source_release.sh
b/tools/create_source_release.sh
similarity index 97%
rename from tools/releasing/create_source_release.sh
rename to tools/create_source_release.sh
index e7beca2..6962e6d 100755
--- a/tools/releasing/create_source_release.sh
+++ b/tools/create_source_release.sh
@@ -22,7 +22,7 @@
# apache-paimon-mosaic-{version}-src.tgz.asc
# apache-paimon-mosaic-{version}-src.tgz.sha512
#
-# Usage: cd tools && RELEASE_VERSION=0.1.0 ./releasing/create_source_release.sh
+# Usage: cd tools && RELEASE_VERSION=0.1.0 ./create_source_release.sh
##
## Variables with defaults (if not overwritten by environment)
diff --git a/tools/dependencies.py b/tools/dependencies.py
new file mode 100755
index 0000000..a4f7be1
--- /dev/null
+++ b/tools/dependencies.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Check and generate Rust dependency license information for ASF release
compliance.
+
+Requires cargo-deny: cargo install cargo-deny
+Requires Python 3.11+ (uses tomllib).
+
+Usage:
+ python3 tools/dependencies.py check # Verify all deps have approved
licenses
+ python3 tools/dependencies.py generate # Generate DEPENDENCIES.rust.tsv
+"""
+
+import sys
+
+if sys.version_info < (3, 11):
+ sys.exit(
+ "This script requires Python 3.11 or newer (uses tomllib). "
+ f"Current: {sys.version}."
+ )
+
+import subprocess
+import tomllib
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from pathlib import Path
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+
+PACKAGES = ["."]
+root_cargo = ROOT_DIR / "Cargo.toml"
+if root_cargo.exists():
+ with open(root_cargo, "rb") as f:
+ data = tomllib.load(f)
+ members = data.get("workspace", {}).get("members", [])
+ if isinstance(members, list):
+ for m in members:
+ if isinstance(m, str) and m:
+ PACKAGES.append(m)
+
+
+def check_single_package(root):
+ pkg_dir = ROOT_DIR / root if root != "." else ROOT_DIR
+ if (pkg_dir / "Cargo.toml").exists():
+ print(f"Checking dependencies of {root}")
+ subprocess.run(
+ ["cargo", "deny", "check", "license"],
+ cwd=pkg_dir,
+ check=True,
+ )
+ else:
+ print(f"Skipping {root} as Cargo.toml does not exist")
+
+
+def check_deps():
+ for d in PACKAGES:
+ check_single_package(d)
+
+
+def generate_single_package(root):
+ pkg_dir = ROOT_DIR / root if root != "." else ROOT_DIR
+ if (pkg_dir / "Cargo.toml").exists():
+ print(f"Generating dependencies for {root}")
+ result = subprocess.run(
+ ["cargo", "deny", "list", "-f", "tsv", "-t", "0.6"],
+ cwd=pkg_dir,
+ capture_output=True,
+ text=True,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(
+ f"cargo deny list failed in {root}: {result.stderr or
result.stdout}"
+ )
+ out_file = pkg_dir / "DEPENDENCIES.rust.tsv"
+ out_file.write_text(result.stdout)
+ print(f" Written to {out_file}")
+ else:
+ print(f"Skipping {root} as Cargo.toml does not exist")
+
+
+def generate_deps():
+ for d in PACKAGES:
+ generate_single_package(d)
+
+
+if __name__ == "__main__":
+ parser = ArgumentParser(
+ description="Check and generate Rust dependency license information",
+ formatter_class=ArgumentDefaultsHelpFormatter,
+ )
+ parser.set_defaults(func=parser.print_help)
+ subparsers = parser.add_subparsers()
+
+ parser_check = subparsers.add_parser(
+ "check", description="Check dependencies", help="Check dependency
licenses"
+ )
+ parser_check.set_defaults(func=check_deps)
+
+ parser_generate = subparsers.add_parser(
+ "generate",
+ description="Generate dependencies",
+ help="Generate DEPENDENCIES.rust.tsv",
+ )
+ parser_generate.set_defaults(func=generate_deps)
+
+ args = parser.parse_args()
+ arg_dict = dict(vars(args))
+ del arg_dict["func"]
+ args.func(**arg_dict)
diff --git a/tools/releasing/update_branch_version.sh
b/tools/update_branch_version.sh
similarity index 100%
rename from tools/releasing/update_branch_version.sh
rename to tools/update_branch_version.sh
diff --git a/tools/validate_asf_yaml.py b/tools/validate_asf_yaml.py
new file mode 100755
index 0000000..ab8de67
--- /dev/null
+++ b/tools/validate_asf_yaml.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Validate .asf.yaml against known ASF infrastructure schema.
+
+Reference: https://github.com/apache/infrastructure-asfyaml/blob/main/README.md
+"""
+
+import sys
+
+try:
+ import yaml
+except ImportError:
+ sys.exit(
+ "PyYAML is required: pip install pyyaml\n"
+ "Or use: python3 -c \"import pip; pip.main(['install', 'pyyaml'])\""
+ )
+
+ASF_YAML_PATH = ".asf.yaml"
+
+VALID_TOP_LEVEL_KEYS = {
+ "github",
+ "notifications",
+ "staging",
+ "publish",
+ "pelican",
+}
+
+VALID_GITHUB_KEYS = {
+ "description",
+ "homepage",
+ "labels",
+ "features",
+ "enabled_merge_buttons",
+ "protected_branches",
+ "collaborators",
+ "autolinks",
+ "environments",
+ "dependabot_alerts",
+ "dependabot_updates",
+ "code_scanning",
+ "del_branch_on_merge",
+ "ghp_branch",
+ "ghp_path",
+}
+
+VALID_FEATURES_KEYS = {
+ "issues",
+ "discussions",
+ "wiki",
+ "projects",
+}
+
+VALID_MERGE_BUTTON_KEYS = {
+ "squash",
+ "merge",
+ "rebase",
+}
+
+VALID_NOTIFICATIONS_KEYS = {
+ "commits",
+ "issues",
+ "pullrequests",
+ "jira_options",
+ "jobs",
+ "discussions",
+}
+
+
+def validate():
+ errors = []
+
+ try:
+ with open(ASF_YAML_PATH, "r") as f:
+ data = yaml.safe_load(f)
+ except FileNotFoundError:
+ print(f"SKIP: {ASF_YAML_PATH} not found")
+ return 0
+ except yaml.YAMLError as e:
+ print(f"ERROR: Invalid YAML syntax in {ASF_YAML_PATH}: {e}")
+ return 1
+
+ if not isinstance(data, dict):
+ print(f"ERROR: {ASF_YAML_PATH} root must be a mapping")
+ return 1
+
+ for key in data:
+ if key not in VALID_TOP_LEVEL_KEYS:
+ errors.append(f"unexpected top-level key '{key}'")
+
+ github = data.get("github")
+ if isinstance(github, dict):
+ for key in github:
+ if key not in VALID_GITHUB_KEYS:
+ errors.append(f"unexpected key 'github.{key}'")
+
+ features = github.get("features")
+ if isinstance(features, dict):
+ for key in features:
+ if key not in VALID_FEATURES_KEYS:
+ errors.append(
+ f"unexpected key 'github.features.{key}' "
+ f"(allowed: {', '.join(sorted(VALID_FEATURES_KEYS))})"
+ )
+ elif not isinstance(features[key], bool):
+ errors.append(
+ f"'github.features.{key}' must be a boolean, "
+ f"got {type(features[key]).__name__}"
+ )
+
+ merge_buttons = github.get("enabled_merge_buttons")
+ if isinstance(merge_buttons, dict):
+ for key in merge_buttons:
+ if key not in VALID_MERGE_BUTTON_KEYS:
+ errors.append(
+ f"unexpected key 'github.enabled_merge_buttons.{key}' "
+ f"(allowed: {',
'.join(sorted(VALID_MERGE_BUTTON_KEYS))})"
+ )
+
+ notifications = data.get("notifications")
+ if isinstance(notifications, dict):
+ for key in notifications:
+ if key not in VALID_NOTIFICATIONS_KEYS:
+ errors.append(f"unexpected key 'notifications.{key}'")
+
+ if errors:
+ print(f"ERROR: {ASF_YAML_PATH} validation failed:")
+ for err in errors:
+ print(f" - {err}")
+ return 1
+
+ print(f"OK: {ASF_YAML_PATH} is valid")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(validate())
