This is an automated email from the ASF dual-hosted git repository. yuzelin pushed a commit to branch release-0.3 in repository https://gitbox.apache.org/repos/asf/paimon-mosaic.git
commit e4f4930f1c22dd4021092bf0ea9e7e00e67dbd79 Author: jianguotian <[email protected]> AuthorDate: Thu Oct 29 13:48:44 2026 +0800 fix(release): validate artifacts and isolate source candidates (#104) (cherry picked from commit 946f3bb3576d56b3a2d96974d3821409a0efc6c3) --- .github/workflows/release-java.yml | 3 +- .github/workflows/release-vote-gate.yml | 19 +++- docs/creating-a-release.html | 33 +++---- java/pom.xml | 50 +++++++++- tools/create_source_release.sh | 22 ++++- tools/deploy_java_staging.sh | 23 ++++- tools/release_checks.py | 143 +++++++++++++++++++++++++++++ tools/tests/deploy_java_staging_test.sh | 36 +++++++- tools/tests/test_create_source_release.py | 140 +++++++++++++++++++++++++++- tools/tests/test_release_checks.py | 108 ++++++++++++++++++++++ tools/tests/test_release_vote_workflow.py | 29 ++++-- tools/tests/test_verify_binary_artifact.py | 43 +++++++++ tools/verify_binary_artifact.py | 51 +++++++++- 13 files changed, 654 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.yml index ffd4573..d5b17ea 100644 --- a/.github/workflows/release-java.yml +++ b/.github/workflows/release-java.yml @@ -188,7 +188,8 @@ jobs: test -s "$artifact" done - python3 tools/verify_binary_artifact.py --jar "$jar_file" + python3 tools/verify_binary_artifact.py --jar "$jar_file" \ + --sources-jar "$sources_jar" --javadoc-jar "$javadoc_jar" for entry in \ META-INF/DEPENDENCIES \ diff --git a/.github/workflows/release-vote-gate.yml b/.github/workflows/release-vote-gate.yml index 87c0dad..e7739c1 100644 --- a/.github/workflows/release-vote-gate.yml +++ b/.github/workflows/release-vote-gate.yml @@ -29,6 +29,10 @@ on: - "java/src/test/java/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.java" - "tools/create_source_release.sh" - "tools/deploy_java_staging.sh" + - "tools/release_checks.py" + - "tools/verify_binary_artifact.py" + - "tools/tests/test_release_checks.py" + - "tools/tests/test_verify_binary_artifact.py" - "tools/update_branch_version.sh" - "tools/verify_release_versions.py" - "tools/verify_source_archive.py" @@ -49,6 +53,10 @@ on: - "java/src/test/java/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.java" - "tools/create_source_release.sh" - "tools/deploy_java_staging.sh" + - "tools/release_checks.py" + - "tools/verify_binary_artifact.py" + - "tools/tests/test_release_checks.py" + - "tools/tests/test_verify_binary_artifact.py" - "tools/update_branch_version.sh" - "tools/verify_release_versions.py" - "tools/verify_source_archive.py" @@ -77,12 +85,19 @@ jobs: python-version: "3.12" - name: Install test dependencies - run: python -m pip install pytest PyYAML + run: | + sudo apt-get update + sudo apt-get install -y subversion + python -m pip install pytest PyYAML + svn --version --quiet + svnadmin --version --quiet - name: Run release vote tests run: | python -m pytest -q \ tools/tests/test_create_source_release.py \ + tools/tests/test_release_checks.py \ + tools/tests/test_verify_binary_artifact.py \ tools/tests/test_release_vote_workflow.py \ tools/tests/test_update_branch_version.py \ tools/tests/test_verify_release_versions.py \ @@ -117,6 +132,8 @@ jobs: set -euo pipefail python -m compileall -q \ tools/verify_release_versions.py \ + tools/release_checks.py \ + tools/verify_binary_artifact.py \ tools/verify_source_archive.py \ tools/tests/test_create_source_release.py \ tools/tests/test_release_vote_workflow.py \ diff --git a/docs/creating-a-release.html b/docs/creating-a-release.html index eb68767..cad80fc 100644 --- a/docs/creating-a-release.html +++ b/docs/creating-a-release.html @@ -95,6 +95,8 @@ <!-- ============================================================ --> <h2 id="prepare-for-the-release">Prepare for the Release</h2> + <p>Source release checks require <code>python3</code> with <code>tomllib</code> (Python 3.11 or newer). This is a release-tool requirement; it does not change the Python binding's supported runtime versions. Select the required interpreter before starting source packaging.</p> + <h3>One-time Release Manager Setup</h3> <h4>GPG Key</h4> @@ -233,6 +235,7 @@ git push origin ${RC_TAG}</code></pre> <p>If only the Python publish job fails due to a transient PyPI or TestPyPI issue, use <strong>Re-run failed jobs</strong> on the <code>Release</code> workflow. The successful Rust, Java, and Python wheel jobs do not need to be repeated.</p> <h3>Deploy Java Artifacts to Nexus Staging</h3> + <p>Use a complete JDK 8 for Java release packaging. On Apple silicon use its ARM64 build. The script checks the JVM actually used by Maven, uses that same JDK for JNI checks, and builds with the UTC timezone. A JDK 11 release build is rejected because its generated Javadoc adds resources that are not covered by this release's JDK 8 license layout.</p> <p>Check out the exact RC tag and copy the numeric run id from its successful push-triggered <code>Release</code> workflow. Run the local preflight first, then repeat without <code>--dry-run</code>:</p> <pre><code>RELEASE_RUN_ID="12345678901" git checkout ${RC_TAG} @@ -240,27 +243,25 @@ git checkout ${RC_TAG} ./tools/deploy_java_staging.sh --release-version ${RELEASE_VERSION} --rc ${RC_NUM} --run-id ${RELEASE_RUN_ID} --dry-run ./tools/deploy_java_staging.sh --release-version ${RELEASE_VERSION} --rc ${RC_NUM} --run-id ${RELEASE_RUN_ID}</code></pre> <p>The script downloads the four native artifacts and the CI <code>java-package</code>, checks and smoke-tests the CI JAR, then locally rebuilds and signs the Maven artifacts with the Release Manager's GPG and <code>apache.releases.https</code> settings. The CI main JAR is a validation reference and is not directly uploaded to Nexus; the local Maven rebuild is deployed. Record the resulting <code>orgapachepaimon-XXXX</code> repository id and leave it staged for the vote.</p> + <p>The release profile checks the main, sources and Javadoc JARs and verifies all four actual artifact signatures after GPG signing and before Nexus deployment. The signing key or subkey must be RSA with at least 2048 bits, as required by the <a href="https://infra.apache.org/release-distribution.html#sigs-and-sums";>ASF distribution policy</a>; an RSA primary key does not make an Ed25519 signing subkey compliant. The same signature check runs before publishing the local sourc [...] <h3>Create Source Release Artifacts</h3> - <p>Create source release artifacts from the same commit as the RC tag:</p> + <p>Run from the repository root. Create source release artifacts from the same commit as the RC tag and capture the verified output directory:</p> <pre><code>git checkout ${RC_TAG} -cd tools -RELEASE_VERSION=${RELEASE_VERSION} RC_TAG=${RC_TAG} ./create_source_release.sh</code></pre> - <p>This creates the following under <code>tools/release/</code>:</p> +SOURCE_RELEASE_DIR=$(cd tools && RELEASE_VERSION=${RELEASE_VERSION} RC_TAG=${RC_TAG} ./create_source_release.sh)</code></pre> + <p>This creates the following under <code>tools/release/${RC_TAG}/${RC_COMMIT_SHA}/</code>, where <code>RC_COMMIT_SHA</code> is the full commit resolved from the verified signed tag:</p> <ul> <li><code>apache-paimon-mosaic-${RELEASE_VERSION}-src.tgz</code> — source archive</li> <li><code>apache-paimon-mosaic-${RELEASE_VERSION}-src.tgz.asc</code> — GPG signature</li> <li><code>apache-paimon-mosaic-${RELEASE_VERSION}-src.tgz.sha512</code> — SHA-512 checksum</li> </ul> + <p>Build and verification messages go to stderr; stdout contains only the absolute output directory after all checks succeed. A failed run leaves <code>SOURCE_RELEASE_DIR</code> empty, even if it previously held an older candidate's directory. Existing candidate output is never overwritten.</p> <h3>Stage Artifacts to SVN</h3> -<pre><code>svn checkout https://dist.apache.org/repos/dist/dev/paimon/ paimon-dist-dev --depth=immediates -cd paimon-dist-dev -mkdir paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM} -cp ../paimon-mosaic/tools/release/apache-paimon-mosaic-${RELEASE_VERSION}-src.* \ - paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}/ -svn add paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM} -svn commit -m "Add paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM}"</code></pre> + <p>In the same shell, upload only the directory returned by source packaging. The required-variable check stops the command if packaging failed or has not run in this shell. Do not upload the parent <code>tools/release/</code> directory; it may contain older candidates or legacy artifacts.</p> +<pre><code>svn import "${SOURCE_RELEASE_DIR:?Source packaging must succeed first}" \ + "https://dist.apache.org/repos/dist/dev/paimon/paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}"; \ + -m "Add paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM}"</code></pre> <p><strong>Checklist:</strong></p> <ul> @@ -314,14 +315,10 @@ Release Manager</code></pre> <p>If the vote reveals issues:</p> <ol> <li>Fix them on the release branch via normal PRs.</li> - <li>Preserve the previous RC's local source artifacts outside the checkout so the next build does not overwrite them: -<pre><code># From the paimon-mosaic checkout -mv tools/release "../paimon-mosaic-${RC_TAG}-artifacts"</code></pre> - </li> + <li>Keep the previous RC's local source artifacts. The next RC uses a different output directory, so no move or deletion is needed.</li> <li>Remove the old RC from dist dev (optional): -<pre><code>cd paimon-dist-dev -svn remove paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM} -svn commit -m "Remove paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM} (superseded)"</code></pre> +<pre><code>svn remove "https://dist.apache.org/repos/dist/dev/paimon/paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}"; \ + -m "Remove paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM} (superseded)"</code></pre> </li> <li>Drop the Nexus staging repository via the <a href="https://repository.apache.org/#stagingRepositories";>Nexus UI</a>.</li> <li>Increment <code>RC_NUM</code>, then go back to <a href="#build-a-release-candidate">Build a release candidate</a>.</li> diff --git a/java/pom.xml b/java/pom.xml index 643c481..1bb7e61 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -112,6 +112,9 @@ <profiles> <profile> <id>release</id> + <properties> + <gpg.skip>false</gpg.skip> + </properties> <build> <plugins> <plugin> @@ -163,6 +166,7 @@ </goals> <configuration> <bestPractices>true</bestPractices> + <ascDirectory>${project.build.directory}</ascDirectory> <gpgArguments> <arg>--pinentry-mode</arg> <arg>loopback</arg> @@ -171,6 +175,50 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>verify-signed-release-artifacts</id> + <phase>verify</phase> + <goals><goal>run</goal></goals> + <configuration> + <skip>${gpg.skip}</skip> + <target> + <exec executable="python3" failonerror="true"> + <arg value="${project.basedir}/../tools/verify_binary_artifact.py"/> + <arg value="--jar"/> + <arg value="${project.build.directory}/${project.build.finalName}.jar"/> + <arg value="--sources-jar"/> + <arg value="${project.build.directory}/${project.build.finalName}-sources.jar"/> + <arg value="--javadoc-jar"/> + <arg value="${project.build.directory}/${project.build.finalName}-javadoc.jar"/> + </exec> + <exec executable="python3" failonerror="true"> + <arg value="${project.basedir}/../tools/release_checks.py"/> + <arg value="signatures"/> + <arg value="--gpg-executable"/><arg value="${gpg.executable}"/> + <arg value="--gpg-home"/><arg value="${gpg.homedir}"/> + <arg value="--signed-file"/> + <arg value="${project.build.directory}/${project.build.finalName}.pom.asc"/> + <arg value="${project.basedir}/pom.xml"/> + <arg value="--signed-file"/> + <arg value="${project.build.directory}/${project.build.finalName}.jar.asc"/> + <arg value="${project.build.directory}/${project.build.finalName}.jar"/> + <arg value="--signed-file"/> + <arg value="${project.build.directory}/${project.build.finalName}-sources.jar.asc"/> + <arg value="${project.build.directory}/${project.build.finalName}-sources.jar"/> + <arg value="--signed-file"/> + <arg value="${project.build.directory}/${project.build.finalName}-javadoc.jar.asc"/> + <arg value="${project.build.directory}/${project.build.finalName}-javadoc.jar"/> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> @@ -219,7 +267,7 @@ <configuration> <rules> <requireJavaVersion> - <version>1.8.0</version> + <version>[1.8,1.9)</version> </requireJavaVersion> <requireMavenVersion> <version>[3.1.1,)</version> diff --git a/tools/create_source_release.sh b/tools/create_source_release.sh index 539f895..c0f9013 100755 --- a/tools/create_source_release.sh +++ b/tools/create_source_release.sh @@ -17,7 +17,7 @@ # limitations under the License. # -# Create ASF source release artifacts under tools/release/: +# Create ASF source release artifacts under tools/release/{RC_TAG}/{commit}/: # apache-paimon-mosaic-{version}-src.tgz # apache-paimon-mosaic-{version}-src.tgz.asc # apache-paimon-mosaic-{version}-src.tgz.sha512 @@ -25,12 +25,18 @@ # Usage: # cd tools # RELEASE_VERSION=0.3.0 RC_TAG=v0.3.0-rc1 ./create_source_release.sh +# +# Diagnostics go to stderr. Only a successful run prints the verified output +# directory to stdout, for capture by the explicit SVN upload command. # fail immediately set -o errexit set -o nounset set -o pipefail +# Keep stdout empty on every failure, including failures in child commands. +exec 3>&1 1>&2 + # Do not let inherited Git variables redirect checks or replace signature # verification. unset \ @@ -80,6 +86,11 @@ fi cd .. REPOSITORY=$(pwd -P) +if ! python3 -c 'import tomllib' >/dev/null 2>&1; then + echo "Source release checks require python3 with tomllib (Python 3.11+). Select that interpreter as python3 and retry." >&2 + exit 1 +fi + if [[ -n $(git status --porcelain --untracked-files=all) ]]; then echo "The source release must be created from a clean Git worktree" >&2 git status --short >&2 @@ -130,7 +141,7 @@ ARCHIVE="apache-paimon-mosaic-${RELEASE_VERSION}-src.tgz" SIGNATURE="${ARCHIVE}.asc" CHECKSUM="${ARCHIVE}.sha512" ARCHIVE_ROOT="paimon-mosaic-${RELEASE_VERSION}" -RELEASE_DIR="${CURR_DIR}/release" +RELEASE_DIR="${CURR_DIR}/release/${RC_TAG}/${TAG_COMMIT}" if [[ -e "${RELEASE_DIR}" || -L "${RELEASE_DIR}" ]]; then echo "Release output already exists and will not be overwritten: ${RELEASE_DIR}" >&2 @@ -174,6 +185,9 @@ gpg \ --output "${ARTIFACT_DIR}/${SIGNATURE}" \ "${ARTIFACT_DIR}/${ARCHIVE}" +python3 tools/release_checks.py signatures \ + --signed-file "${ARTIFACT_DIR}/${SIGNATURE}" "${ARTIFACT_DIR}/${ARCHIVE}" + if ( cd "${ARTIFACT_DIR}" && if [[ $(uname) == "Darwin" ]]; then @@ -199,6 +213,7 @@ python3 tools/verify_source_archive.py verify \ # The staging and release directories share a filesystem, so this publishes # the fully verified artifact set with one atomic directory rename. +mkdir -p "$(dirname "${RELEASE_DIR}")" python3 -c \ 'import os, sys; os.rename(sys.argv[1], sys.argv[2])' \ "${ARTIFACT_DIR}" \ @@ -208,7 +223,8 @@ trap - EXIT rm -rf "${STAGING_DIR}" echo "" -echo "Source release created successfully. Artifacts in tools/release/:" +echo "Source release created successfully. Artifacts in ${RELEASE_DIR}:" ls -la "${RELEASE_DIR}"/apache-paimon-mosaic-* echo "" echo "Next: upload contents to SVN (see docs/creating-a-release.html)." +printf '%s\n' "${RELEASE_DIR}" >&3 diff --git a/tools/deploy_java_staging.sh b/tools/deploy_java_staging.sh index c79042b..c63f58b 100755 --- a/tools/deploy_java_staging.sh +++ b/tools/deploy_java_staging.sh @@ -248,7 +248,7 @@ else fi check_java_package_inputs_clean() { - local paths=(java LICENSE-binary tools/deploy_java_staging.sh tools/verify_binary_artifact.py) + local paths=(java LICENSE LICENSE-binary tools/deploy_java_staging.sh tools/verify_binary_artifact.py tools/release_checks.py) local untracked if ! git -C "$REPO_DIR" diff --quiet -- "${paths[@]}" || @@ -330,6 +330,16 @@ fi validate_github_run +MAVEN_VERSION=$(cd "$REPO_DIR/java" && "$MVN" --version) +JAVA_HOME=$(printf '%s\n' "$MAVEN_VERSION" | python3 "$SCRIPT_DIR/release_checks.py" java-home) +if command -v cygpath >/dev/null 2>&1; then + JAVA_HOME=$(cygpath -u "$JAVA_HOME") +fi +export JAVA_HOME +export PATH="$JAVA_HOME/bin:$PATH" +export TZ=UTC +export MAVEN_OPTS="${MAVEN_OPTS:-} -Duser.timezone=UTC" + cleanup_download_dir() { if [[ -n "$NATIVE_DIR" ]]; then rm -rf "$NATIVE_DIR" @@ -556,12 +566,17 @@ validate_maven_artifacts() { done for artifact in "$sources_jar" "$ci_sources_jar"; do + python3 "$REPO_DIR/tools/verify_binary_artifact.py" --sources-jar "$artifact" if grep -Eq '^(native/|META-INF/LICENSE-binary$)' <<<"$(jar tf "$artifact")"; then echo "Sources jar contains binary-only resources: $artifact" >&2 exit 1 fi done + for artifact in "$javadoc_jar" "$ci_javadoc_jar"; do + python3 "$REPO_DIR/tools/verify_binary_artifact.py" --javadoc-jar "$artifact" + done + for main_jar in "$jar_file" "$ci_jar" do for entry in \ @@ -588,9 +603,9 @@ validate_maven_artifacts() { echo "Packaged JAR loader smoke test class is missing from test output." >&2 exit 1 fi - java -cp "$jar_file:$test_classes" \ + "$JAVA_HOME/bin/java" -Djava.library.path=/nonexistent-mosaic-release-check -cp "$jar_file:$test_classes" \ org.apache.paimon.mosaic.MosaicNativeLoaderSmokeTest - java -cp "$ci_jar:$test_classes" \ + "$JAVA_HOME/bin/java" -Djava.library.path=/nonexistent-mosaic-release-check -cp "$ci_jar:$test_classes" \ org.apache.paimon.mosaic.MosaicNativeLoaderSmokeTest } @@ -613,7 +628,7 @@ echo "" if [[ "$DRY_RUN" == "true" ]]; then echo "Java staging dry run finished successfully." else - DEPLOY_CMD=("${MVN_BASE_CMD[@]}" deploy -Prelease "-DstagingDescription=$STAGING_DESCRIPTION") + DEPLOY_CMD=("${MVN_BASE_CMD[@]}" deploy -Prelease "-DstagingDescription=$STAGING_DESCRIPTION" -Dgpg.skip=false) if [[ "$SKIP_TESTS" == "true" ]]; then DEPLOY_CMD+=(-DskipTests) fi diff --git a/tools/release_checks.py b/tools/release_checks.py new file mode 100644 index 0000000..6614b35 --- /dev/null +++ b/tools/release_checks.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check the local release JVM and actual artifact signatures before upload.""" + +import argparse +import os +from pathlib import Path +import re +import subprocess +import sys + + +def output(command, **kwargs): + result = subprocess.run(command, capture_output=True, text=True, **kwargs) + if result.returncode: + raise ValueError(result.stderr.strip() or result.stdout.strip()) + return result.stdout + result.stderr + + +def java_home(version): + version = re.sub(r"\x1b\[[0-9;]*m", "", version) + if not re.search(r"Java version: 1\.8\.", version): + raise ValueError("Java release packaging requires Maven running on JDK 8.\n" + version) + runtime = re.search(r"(?:runtime:|Java home:)\s*([^\r\n]+)", version) + if runtime is None: + raise ValueError("Cannot determine Maven's Java home from mvn --version") + home = Path(runtime[1].strip()).resolve() + if home.name == "jre": + home = home.parent + suffix = ".exe" if os.name == "nt" else "" + tools = ("java", "javac", "javadoc", "jar") + if not all((home / "bin" / (tool + suffix)).is_file() for tool in tools): + raise ValueError("Maven must use a complete JDK 8: " + str(home)) + settings = output([str(home / "bin" / ("java" + suffix)), "-XshowSettings:properties", "-version"]) + properties = dict(re.findall(r"(?m)^\s*(java.version|os.name|os.arch)\s*=\s*(.+?)\s*$", settings)) + if not properties.get("java.version", "").startswith("1.8."): + raise ValueError("Maven's Java home does not contain JDK 8") + name = properties.get("os.name", "").lower() + arch = properties.get("os.arch", "").lower() + arch = {"amd64": "x86_64", "arm64": "aarch64"}.get(arch, arch) + if "mac" in name or "darwin" in name: + platform = "macos" + elif "win" in name: + platform = "windows" + else: + platform = name + supported = { + ("linux", "x86_64"), ("linux", "aarch64"), + ("macos", "aarch64"), ("windows", "x86_64"), + } + if (platform, arch) not in supported: + raise ValueError( + "No release JNI library for Maven's JVM: " + platform + "/" + arch + + ". On Apple silicon select an ARM64 JDK 8." + ) + print("Verified Maven JDK 8: " + str(home) + " (" + platform + "/" + arch + ")", file=sys.stderr) + return home + + +def gpg_command(executable=None, homedir=None): + # Maven leaves an unresolved expression literal when the optional property + # is unset. In that case use GPG's normal executable/home (including GNUPGHOME). + command = [executable if executable and executable != "${gpg.executable}" else "gpg", "--batch"] + if homedir and homedir != "${gpg.homedir}": + command += ["--homedir", homedir] + return command + + +def verify_signatures(signed_files, executable=None, homedir=None): + gpg = gpg_command(executable, homedir) + for signature, artifact in signed_files: + signature, artifact = Path(signature), Path(artifact) + if not signature.read_bytes().lstrip().startswith(b"-----BEGIN PGP SIGNATURE-----"): + raise ValueError("Expected an ASCII-armored detached signature: " + str(signature)) + result = output(gpg + ["--status-fd", "1", "--verify", str(signature), str(artifact)]) + signatures = [line.split() for line in result.splitlines() if line.startswith("[GNUPG:] VALIDSIG ")] + invalid = ("[GNUPG:] EXPKEYSIG ", "[GNUPG:] EXPSIG ", "[GNUPG:] REVKEYSIG ") + if not signatures or any(token in result for token in invalid): + raise ValueError("No valid current signature: " + str(signature)) + for status in signatures: + fingerprint = status[2] + listing = output(gpg + ["--with-colons", "--fixed-list-mode", "--list-keys", fingerprint]) + keys = {} + key = None + for line in listing.splitlines(): + fields = line.split(":") + if fields[0] in ("pub", "sub"): + key = (int(fields[3]), int(fields[2])) + elif fields[0] == "fpr" and key is not None: + keys[fields[9]] = key + key = None + algorithm, bits = keys.get(fingerprint, (0, 0)) + # Check the actual signing subkey, not merely its primary key. + if int(status[8]) not in (1, 3) or algorithm not in (1, 3) or bits < 2048: + raise ValueError( + "ASF release signatures require RSA >=2048 bits; actual signer " + + fingerprint + " uses algorithm " + str(algorithm) + + ", " + str(bits) + " bits: " + str(artifact) + ) + print("Verified RSA release signature: " + str(artifact) + " (" + fingerprint + ")") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("java-home", help="Read Maven --version from stdin") + signatures = subparsers.add_parser("signatures") + signatures.add_argument( + "--signed-file", nargs=2, action="append", required=True, + metavar=("SIGNATURE", "ARTIFACT"), + ) + signatures.add_argument("--gpg-executable") + signatures.add_argument("--gpg-home") + args = parser.parse_args() + try: + if args.command == "java-home": + print(java_home(sys.stdin.read())) + else: + verify_signatures(args.signed_file, args.gpg_executable, args.gpg_home) + except (OSError, ValueError, IndexError) as error: + print("Release check failed: " + str(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/deploy_java_staging_test.sh b/tools/tests/deploy_java_staging_test.sh index 1683df5..06ac857 100755 --- a/tools/tests/deploy_java_staging_test.sh +++ b/tools/tests/deploy_java_staging_test.sh @@ -24,6 +24,7 @@ set -o pipefail SOURCE_REPO=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) REAL_PYTHON=$(command -v python3) TEST_ROOT=$(mktemp -d) +TEST_ROOT=$(cd "$TEST_ROOT" && pwd -P) trap 'rm -rf "$TEST_ROOT"' EXIT TESTS=0 FIXTURES=0 @@ -59,6 +60,7 @@ new_fixture() { "$MOCK_BIN" \ "$STAGE_TMPDIR" cp "$SOURCE_REPO/tools/deploy_java_staging.sh" "$FIXTURE/tools/" + cp "$SOURCE_REPO/tools/release_checks.py" "$FIXTURE/tools/" cp "$SOURCE_REPO/java/src/test/java/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.java" \ "$FIXTURE/java/src/test/java/org/apache/paimon/mosaic/" cat > "$FIXTURE/java/pom.xml" <<'POM' @@ -114,7 +116,12 @@ MOCK cat > "$MOCK_BIN/mvn" <<'MOCK' #!/usr/bin/env bash set -euo pipefail +if [[ "$*" == --version ]]; then + printf 'Apache Maven 3.8.8\nJava version: %s, vendor: Test, runtime: %s/jre\n' "${MOCK_MAVEN_JAVA_VERSION:-1.8.0_502}" "$MOCK_BIN" + exit 0 +fi printf 'mvn %s\n' "$*" >> "$MOCK_LOG" +printf 'environment %s %s\n' "$JAVA_HOME" "$TZ" >> "$MOCK_LOG" mkdir -p target/test-classes/org/apache/paimon/mosaic printf class > target/test-classes/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.class if [[ "${OMIT_LOCAL_MAIN:-0}" != 1 ]]; then @@ -161,8 +168,15 @@ MOCK cat > "$MOCK_BIN/java" <<'MOCK' #!/usr/bin/env bash set -euo pipefail +if [[ "${1:-}" == -XshowSettings:properties ]]; then + printf ' java.version = 1.8.0_502\n os.name = Mac OS X\n os.arch = %s\n' "${MOCK_JAVA_ARCH:-aarch64}" + exit 0 +fi printf 'java %s\n' "$*" >> "$MOCK_LOG" MOCK + mkdir -p "$MOCK_BIN/bin" + for tool in java jar; do ln -s "../$tool" "$MOCK_BIN/bin/$tool"; done + for tool in javac javadoc; do printf '#!/bin/sh\nexit 0\n' > "$MOCK_BIN/bin/$tool"; chmod +x "$MOCK_BIN/bin/$tool"; done cat > "$MOCK_BIN/file" <<'MOCK' #!/usr/bin/env bash set -euo pipefail @@ -202,6 +216,7 @@ run_stage() { PATH="$MOCK_BIN:$PATH" \ TMPDIR="$STAGE_TMPDIR" \ MOCK_LOG="$MOCK_LOG" \ + MOCK_BIN="$MOCK_BIN" \ MOCK_RUN_SHA="${MOCK_RUN_SHA:-$HEAD_SHA}" \ MOCK_TAG=v1.2.3-rc1 \ REAL_PYTHON="$REAL_PYTHON" \ @@ -238,12 +253,13 @@ assert_contains "$MOCK_LOG" "--dir $STAGE_TMPDIR/paimon-mosaic-java-staging." assert_contains "$MOCK_LOG" "mvn clean verify -Prelease -Dgpg.skip=true -DskipTests" assert_not_contains "$MOCK_LOG" "mvn deploy" [[ $(grep -c '^java ' "$MOCK_LOG") -eq 2 ]] || fail "dry-run must smoke local and CI JARs" -[[ $(grep -c '^python3 .*verify_binary_artifact.py' "$MOCK_LOG") -eq 2 ]] || - fail "dry-run must verify local and CI JAR contents" +[[ $(grep -c '^python3 .*verify_binary_artifact.py' "$MOCK_LOG") -eq 6 ]] || + fail "dry-run must verify local and CI main, sources and Javadoc JAR contents" assert_stage_cleanup [[ ! -e "$FIXTURE/java/src/main/resources/native" ]] || fail "default cleanup must remove staged Java resources" -pass "successful dry-run validates all five artifacts and both JARs, and removes staging downloads" +assert_contains "$MOCK_LOG" "environment $MOCK_BIN UTC" +pass "successful dry-run validates all five artifacts, all JAR types, and uses Maven's JDK and UTC" new_fixture settings="$FIXTURE/settings.xml" @@ -309,4 +325,18 @@ assert_not_contains "$MOCK_LOG" "mvn deploy" assert_stage_cleanup pass "failing runs remove staging downloads and block deploy" +new_fixture +export MOCK_MAVEN_JAVA_VERSION=11.0.1 +if run_stage --dry-run > "$FIXTURE/output" 2>&1; then fail "JDK11 must be rejected"; fi +assert_contains "$FIXTURE/output" "Maven running on JDK 8" +assert_not_contains "$MOCK_LOG" "run download" +unset MOCK_MAVEN_JAVA_VERSION +new_fixture +export MOCK_JAVA_ARCH=x86_64 +if run_stage --dry-run > "$FIXTURE/output" 2>&1; then fail "Intel macOS JVM must be rejected before build"; fi +assert_contains "$FIXTURE/output" "macos/x86_64" +assert_not_contains "$MOCK_LOG" "run download" +unset MOCK_JAVA_ARCH +pass "unsupported Maven JDK and JVM architecture fail before downloading artifacts" + echo "PASS: $TESTS focused Java staging tests" diff --git a/tools/tests/test_create_source_release.py b/tools/tests/test_create_source_release.py index 0f6e4dc..1132c87 100644 --- a/tools/tests/test_create_source_release.py +++ b/tools/tests/test_create_source_release.py @@ -18,8 +18,10 @@ from __future__ import annotations import hashlib +import html import os from pathlib import Path +import re import shutil import subprocess import sys @@ -73,7 +75,7 @@ def signing_home(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, str]: "", "--quick-gen-key", "Release Test <[email protected]>", - "rsa1024", + "rsa2048", "sign", "0", ], @@ -108,6 +110,7 @@ def initialize_release_repo( write(tools / SOURCE_SCRIPT.name, source_script) shutil.copy2(SOURCE_VERIFIER, tools / SOURCE_VERIFIER.name) shutil.copy2(VERSION_VERIFIER, tools / VERSION_VERIFIER.name) + shutil.copy2(ROOT / "tools/release_checks.py", tools / "release_checks.py") write( repo / "Cargo.toml", @@ -183,7 +186,7 @@ def run_script( repo: Path, env: dict[str, str], ) -> subprocess.CompletedProcess[bytes]: - return subprocess.run( + result = subprocess.run( ["bash", SOURCE_SCRIPT.name], cwd=repo / "tools", env=env, @@ -191,16 +194,25 @@ def run_script( stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) + if result.returncode: + assert result.stdout == b"", output(result) + return result def output(result: subprocess.CompletedProcess[bytes]) -> str: return (result.stdout + result.stderr).decode("utf-8", errors="replace") -def artifact_paths(repo: Path) -> tuple[Path, Path, Path]: +def artifact_paths( + repo: Path, rc_tag: str = RC_TAG, commit: str | None = None, +) -> tuple[Path, Path, Path]: + if commit is None: + commit = run(["git", "rev-parse", "HEAD"], cwd=repo).stdout.decode().strip() archive = ( repo / "tools/release" + / rc_tag + / commit / f"apache-paimon-mosaic-{VERSION}-src.tgz" ) return archive, Path(f"{archive}.asc"), Path(f"{archive}.sha512") @@ -216,6 +228,8 @@ def test_creates_verified_artifacts_from_signed_head_tag( assert result.returncode == 0, output(result) archive, signature, checksum = artifact_paths(repo) + assert result.stdout.decode() == str(archive.parent) + "\n" + assert set(archive.parent.iterdir()) == {archive, signature, checksum} assert archive.is_file() assert signature.is_file() assert checksum.is_file() @@ -242,6 +256,21 @@ def test_creates_verified_artifacts_from_signed_head_tag( ) +def test_missing_tomllib_fails_before_creating_any_archive(tmp_path, signing_home): + repo, env, _ = initialize_release_repo(tmp_path, signing_home) + binary = tmp_path / "bin" + binary.mkdir() + python = binary / "python3" + python.write_text("#!/bin/sh\nexit 1\n") + python.chmod(0o755) + env["PATH"] = str(binary) + os.pathsep + env["PATH"] + result = run_script(repo, env) + assert result.returncode != 0 + assert "python3 with tomllib" in output(result) + assert not (repo / "tools/release").exists() + assert not list((repo / "tools").glob(".source-release.*")) + + @pytest.mark.parametrize("missing", ("RELEASE_VERSION", "RC_TAG")) def test_requires_release_version_and_rc_tag( tmp_path: Path, @@ -329,6 +358,111 @@ def test_existing_artifact_is_not_overwritten( assert not checksum.exists() +def next_candidate(repo, env, signing_home, *, mismatched_version=False): + if mismatched_version: + write( + repo / "python/pyproject.toml", + '[project]\nname = "paimon-mosaic"\nversion = "0.3.1"\n', + ) + write(repo / "README.md", "next candidate\n") + run(["git", "add", "."], cwd=repo) + run(["git", "commit", "-q", "-m", "next candidate"], cwd=repo) + env["RC_TAG"] = f"v{VERSION}-rc2" + run( + ["git", "tag", "-s", "-u", signing_home[1], "-m", "RC2", env["RC_TAG"]], + cwd=repo, env=env, + ) + + +def test_next_candidate_preserves_previous_and_legacy_artifacts(tmp_path, signing_home): + repo, env, _ = initialize_release_repo(tmp_path, signing_home) + assert run_script(repo, env).returncode == 0 + previous = artifact_paths(repo) + previous_bytes = [path.read_bytes() for path in previous] + legacy = repo / "tools/release/apache-paimon-mosaic-0.2.0-src.tgz" + legacy.write_bytes(b"legacy source package") + + next_candidate(repo, env, signing_home) + result = run_script(repo, env) + + assert result.returncode == 0, output(result) + archive, signature, checksum = artifact_paths(repo, env["RC_TAG"]) + assert result.stdout.decode() == str(archive.parent) + "\n" + assert set(archive.parent.iterdir()) == {archive, signature, checksum} + assert [path.read_bytes() for path in previous] == previous_bytes + assert legacy.read_bytes() == b"legacy source package" + with tarfile.open(archive, "r:gz") as source: + assert source.pax_headers["comment"] == archive.parent.name + assert source.extractfile(f"paimon-mosaic-{VERSION}/README.md").read() == b"next candidate\n" + + [email protected]("failure", (None, "versions", "signing", "checksum", "python", "existing")) +def test_documented_native_svn_upload_cannot_reuse_old_output(tmp_path, signing_home, failure): + if not all(shutil.which(tool) for tool in ("svn", "svnadmin")): + pytest.skip("svn and svnadmin are required for the local upload integration test") + repo, env, _ = initialize_release_repo(tmp_path / "checkout with spaces", signing_home) + assert run_script(repo, env).returncode == 0 + previous = artifact_paths(repo) + previous_bytes = [path.read_bytes() for path in previous] + next_candidate(repo, env, signing_home, mismatched_version=failure == "versions") + env["RC_NUM"] = "2" + # Simulate a shell left over from a successful earlier release attempt. + env["SOURCE_RELEASE_DIR"] = str(previous[0].parent) + if failure == "existing": + assert run_script(repo, env).returncode == 0 + env["SOURCE_RELEASE_DIR"] = str(artifact_paths(repo, env["RC_TAG"])[0].parent) + if failure in ("signing", "checksum", "python"): + binary = tmp_path / "bin" + binary.mkdir() + if failure == "signing": + env["REAL_GPG"] = shutil.which("gpg") + command, body = "gpg", ( + 'for arg in "$@"; do\n' + ' if [ "$arg" = "--detach-sign" ]; then exit 42; fi\n' + 'done\nexec "$REAL_GPG" "$@"\n' + ) + else: + command = "python3" if failure == "python" else ( + "shasum" if sys.platform == "darwin" else "sha512sum" + ) + body = "exit 42\n" + wrapper = binary / command + write(wrapper, "#!/bin/sh\n" + body) + wrapper.chmod(0o755) + env["PATH"] = str(binary) + os.pathsep + env["PATH"] + + svn_repo = tmp_path / "svn-repository" + run(["svnadmin", "create", str(svn_repo)], cwd=tmp_path) + svn_url = svn_repo.as_uri() + documentation = (ROOT / "docs/creating-a-release.html").read_text() + blocks = [html.unescape(block) for block in re.findall( + r"<pre><code>(.*?)</code></pre>", documentation, re.DOTALL, + )] + preparation = next(line for block in blocks for line in block.splitlines() + if line.startswith("SOURCE_RELEASE_DIR=")) + upload = next(block for block in blocks if block.startswith("svn import ")) + upload = upload.replace("https://dist.apache.org/repos/dist/dev/paimon";, svn_url) + # Intentionally no `set -e` or &&: execute the actual documented upload + # even after packaging fails, just as separately pasted RM commands do. + result = subprocess.run( + ["bash", "-c", preparation + "\n" + upload], + cwd=repo, env=env, capture_output=True, + ) + if failure: + assert result.returncode != 0, output(result) + assert "Source packaging must succeed first" in output(result) + assert run(["svn", "list", svn_url], cwd=repo).stdout == b"" + else: + assert result.returncode == 0, output(result) + exported = tmp_path / "exported" + run(["svn", "export", f"{svn_url}/paimon-mosaic-{VERSION}-rc2", str(exported)], cwd=repo) + artifacts = artifact_paths(repo, env["RC_TAG"]) + assert {path.name for path in exported.iterdir()} == {path.name for path in artifacts} + for artifact in artifacts: + assert (exported / artifact.name).read_bytes() == artifact.read_bytes() + assert [path.read_bytes() for path in previous] == previous_bytes + + def test_release_version_must_match_rc_tag( tmp_path: Path, signing_home: tuple[Path, str], diff --git a/tools/tests/test_release_checks.py b/tools/tests/test_release_checks.py new file mode 100644 index 0000000..7176d55 --- /dev/null +++ b/tools/tests/test_release_checks.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path +import shutil +import subprocess +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import release_checks as checks # noqa: E402 + + [email protected](scope="module") +def signing_keys(tmp_path_factory): + if shutil.which("gpg") is None: + pytest.skip("gpg is required") + home = tmp_path_factory.mktemp("signature-keys") + home.chmod(0o700) + command = ["gpg", "--homedir", str(home), "--batch", "--pinentry-mode", "loopback", "--passphrase", ""] + + def run(*args): + return subprocess.check_output(command + list(args), stderr=subprocess.PIPE, text=True) + + def fingerprints(): + return [line.split(":")[9] for line in run("--with-colons", "--list-secret-keys").splitlines() if line.startswith("fpr:")] + + run("--quick-gen-key", "Release Policy Test <[email protected]>", "rsa2048", "cert", "0") + primary = fingerprints()[0] + keys = {} + for algorithm in ("rsa2048", "rsa1024", "ed25519"): + before = set(fingerprints()) + run("--quick-add-key", primary, algorithm, "sign", "0") + keys[algorithm] = next(iter(set(fingerprints()) - before)) + yield home, command, keys + subprocess.run(["gpgconf", "--homedir", str(home), "--kill", "gpg-agent"], capture_output=True) + + [email protected]("algorithm", ["rsa2048", "rsa1024", "ed25519"]) +def test_checks_actual_signing_subkey(signing_keys, tmp_path, algorithm): + home, command, keys = signing_keys + artifact = tmp_path / "artifact" + artifact.write_bytes(b"release bytes\n") + signature = Path(str(artifact) + ".asc") + subprocess.run(command + ["--armor", "--local-user", keys[algorithm] + "!", "--detach-sign", "--output", str(signature), str(artifact)], check=True, capture_output=True) + if algorithm == "rsa2048": + checks.verify_signatures([(signature, artifact)], homedir=str(home)) + else: + # The primary key is RSA2048 in all three cases. Checking it alone + # would incorrectly approve these weak/non-RSA signing subkeys. + with pytest.raises(ValueError, match="RSA >=2048"): + checks.verify_signatures([(signature, artifact)], homedir=str(home)) + + +def test_tampered_signed_artifact_is_rejected(signing_keys, tmp_path): + home, command, keys = signing_keys + artifact = tmp_path / "artifact" + artifact.write_bytes(b"original") + signature = Path(str(artifact) + ".asc") + subprocess.run(command + ["--armor", "--local-user", keys["rsa2048"] + "!", "--detach-sign", "--output", str(signature), str(artifact)], check=True, capture_output=True) + artifact.write_bytes(b"modified") + with pytest.raises(ValueError): + checks.verify_signatures([(signature, artifact)], homedir=str(home)) + + +def test_binary_signature_is_rejected_before_verification(tmp_path): + signature = tmp_path / "binary.asc" + signature.write_bytes(b"\x89binary") + with pytest.raises(ValueError, match="ASCII-armored"): + checks.verify_signatures([(signature, tmp_path / "artifact")]) + + +def test_unset_maven_properties_use_gpg_defaults(): + assert checks.gpg_command("${gpg.executable}", "${gpg.homedir}") == ["gpg", "--batch"] + + +def test_java_home_uses_maven_jdk_instead_of_path_java(tmp_path, monkeypatch): + jdk = tmp_path / "Maven JDK" + (jdk / "bin").mkdir(parents=True) + for name in ("java", "javac", "javadoc", "jar"): + (jdk / "bin" / name).touch() + commands = [] + + def output(command, **kwargs): + commands.append(command) + assert command[0] == str(jdk / "bin/java") + return "java.version = 1.8.0_502\nos.name = Mac OS X\nos.arch = aarch64\n" + + monkeypatch.setattr(checks, "output", output) + monkeypatch.setenv("JAVA_HOME", "/wrong/path/jdk11") + version = f"Java version: 1.8.0_502, vendor: Test, runtime: {jdk}/jre\n" + assert checks.java_home(version) == jdk + assert len(commands) == 1 diff --git a/tools/tests/test_release_vote_workflow.py b/tools/tests/test_release_vote_workflow.py index 091393b..91ab760 100644 --- a/tools/tests/test_release_vote_workflow.py +++ b/tools/tests/test_release_vote_workflow.py @@ -67,6 +67,10 @@ REQUIRED_GATE_PATHS = { "java/src/test/java/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.java", "tools/create_source_release.sh", "tools/deploy_java_staging.sh", + "tools/release_checks.py", + "tools/verify_binary_artifact.py", + "tools/tests/test_release_checks.py", + "tools/tests/test_verify_binary_artifact.py", "tools/update_branch_version.sh", "tools/verify_release_versions.py", "tools/verify_source_archive.py", @@ -113,6 +117,8 @@ python3 tools/verify_source_archive.py verify \\ """ GATE_TEST_COMMAND = """python -m pytest -q \\ tools/tests/test_create_source_release.py \\ + tools/tests/test_release_checks.py \\ + tools/tests/test_verify_binary_artifact.py \\ tools/tests/test_release_vote_workflow.py \\ tools/tests/test_update_branch_version.py \\ tools/tests/test_verify_release_versions.py \\ @@ -137,6 +143,8 @@ python3 tools/verify_source_archive.py verify \\ GATE_STATIC_COMMAND = """set -euo pipefail python -m compileall -q \\ tools/verify_release_versions.py \\ + tools/release_checks.py \\ + tools/verify_binary_artifact.py \\ tools/verify_source_archive.py \\ tools/tests/test_create_source_release.py \\ tools/tests/test_release_vote_workflow.py \\ @@ -243,7 +251,13 @@ def assert_gate_contract(workflow: dict) -> None: assert "if" not in step assert "continue-on-error" not in step - assert install_step["run"] == "python -m pip install pytest PyYAML" + assert install_step["run"].splitlines() == [ + "sudo apt-get update", + "sudo apt-get install -y subversion", + "python -m pip install pytest PyYAML", + "svn --version --quiet", + "svnadmin --version --quiet", + ] assert test_step["run"] == GATE_TEST_COMMAND assert staging_step["shell"] == "bash" assert staging_step["run"] == "bash tools/tests/deploy_java_staging_test.sh" @@ -732,19 +746,14 @@ def test_release_documentation_describes_source_archive_preflight() -> None: assert "temporary source archive" in source -def test_release_documentation_preserves_previous_rc_output_before_retry() -> None: +def test_release_documentation_keeps_separate_candidate_outputs() -> None: source = RELEASE_DOCUMENTATION.read_text(encoding="utf-8") retry_section = source.partition('<h2 id="fix-any-issues">')[2].partition( '<h2 id="finalize-the-release">' )[0] - preserve_command = ( - 'mv tools/release "../paimon-mosaic-${RC_TAG}-artifacts"' - ) - - assert preserve_command in retry_section - assert retry_section.index(preserve_command) < retry_section.index( - "Increment <code>RC_NUM</code>" - ) + assert "Keep the previous RC's local source artifacts" in retry_section + assert 'tools/release/${RC_TAG}/${RC_COMMIT_SHA}/' in source + assert 'mv tools/release' not in retry_section def test_release_vote_gate_matches_blocking_contract() -> None: diff --git a/tools/tests/test_verify_binary_artifact.py b/tools/tests/test_verify_binary_artifact.py index deb54bc..6501e49 100644 --- a/tools/tests/test_verify_binary_artifact.py +++ b/tools/tests/test_verify_binary_artifact.py @@ -35,6 +35,12 @@ def repository(tmp_path, monkeypatch): "LICENSE-binary": b"JNI third-party licenses\n", "LICENSE-binary-ffi": b"FFI third-party licenses\n", "java/src/main/binary-resources/META-INF/NOTICE": b"Binary notices\n", + "java/pom.xml": ( + b'<project xmlns="http://maven.apache.org/POM/4.0.0";>' + b'<inceptionYear>2026</inceptionYear><properties>' + b'<project.build.outputTimestamp>2026-01-01T00:00:00Z' + b'</project.build.outputTimestamp></properties></project>' + ), }.items(): path = root / name path.parent.mkdir(parents=True, exist_ok=True) @@ -127,3 +133,40 @@ def test_rejects_stale_java_license_before_resolving_dependencies(repository): path.write_bytes(b"Apache license\n") with pytest.raises(ValueError, match="Java binary LICENSE is stale"): verifier.verify_source_legal_inventory() + + +def auxiliary_entries(): + return { + "META-INF/LICENSE": b"Apache license\n", + "META-INF/NOTICE": b"Mosaic\nCopyright 2026 The Apache Software Foundation\n", + "META-INF/MANIFEST.MF": b"Manifest-Version: 1.0\r\nBuild-Jdk-Spec: 1.8\r\n", + "script.js": b"JDK8 script", + "stylesheet.css": b"JDK8 stylesheet", + } + + [email protected]("javadoc", [False, True]) +def test_accepts_java_auxiliary_license_and_utc_notice(repository, tmp_path, javadoc): + artifact = tmp_path / "auxiliary.jar" + with zipfile.ZipFile(artifact, "w") as z: + for name, data in auxiliary_entries().items(): + z.writestr(name, data) + verifier.verify_java_auxiliary(artifact, javadoc=javadoc) + + [email protected]("entry,data,message", [ + ("META-INF/NOTICE", b"Copyright 2025 The Apache Software Foundation", "UTC copyright"), + ("META-INF/LICENSE", b"incomplete", "canonical Apache LICENSE"), + ("META-INF/MANIFEST.MF", b"Build-Jdk-Spec: 11", "generated with JDK 8"), + ("resources/jquery.js", b"extra third party library", "license review"), + ("native/linux/x86_64/library.so", b"binary", "binary-only"), +]) +def test_rejects_invalid_java_auxiliary_artifact(repository, tmp_path, entry, data, message): + entries = auxiliary_entries() + entries[entry] = data + artifact = tmp_path / "bad.jar" + with zipfile.ZipFile(artifact, "w") as z: + for name, value in entries.items(): + z.writestr(name, value) + with pytest.raises(ValueError, match=message): + verifier.verify_java_auxiliary(artifact, javadoc=True) diff --git a/tools/verify_binary_artifact.py b/tools/verify_binary_artifact.py index bde4096..841bcef 100644 --- a/tools/verify_binary_artifact.py +++ b/tools/verify_binary_artifact.py @@ -20,10 +20,12 @@ """Verify legal files and native resources in binary convenience artifacts.""" import argparse +from datetime import datetime, timezone import glob import json import subprocess import sys +import xml.etree.ElementTree as ET import zipfile from pathlib import Path @@ -152,6 +154,44 @@ def verify_jar(artifact, native_files): print(f"Verified Java JAR: {artifact}") +def verify_java_auxiliary(artifact, javadoc=False): + pom = ET.parse(ROOT / "java/pom.xml") + ns = {"m": "http://maven.apache.org/POM/4.0.0"} + timestamp = pom.findtext("m:properties/m:project.build.outputTimestamp", namespaces=ns) + if timestamp is None: + raise ValueError("Java release POM must define project.build.outputTimestamp") + if timestamp.isdigit(): + end = datetime.fromtimestamp(int(timestamp), timezone.utc).year + else: + end = datetime.fromisoformat(timestamp.replace("Z", "+00:00")).astimezone(timezone.utc).year + start = pom.findtext("m:inceptionYear", str(end), namespaces=ns) + years = start if start == str(end) else f"{start}-{end}" + with zipfile.ZipFile(artifact) as archive: + require_entries(archive, artifact, ("META-INF/LICENSE", "META-INF/NOTICE")) + if archive.read("META-INF/LICENSE").strip() != (ROOT / "LICENSE").read_bytes().strip(): + raise ValueError(f"{artifact} must contain the canonical Apache LICENSE") + notice = archive.read("META-INF/NOTICE").decode("utf-8") + if f"Copyright {years} The Apache Software Foundation" not in notice: + raise ValueError(f"{artifact} NOTICE must use UTC copyright years {years}") + names = archive.namelist() + if any(name.startswith("native/") or name == "META-INF/LICENSE-binary" for name in names): + raise ValueError(f"{artifact} contains binary-only resources") + if javadoc: + require_entries(archive, artifact, ("META-INF/MANIFEST.MF",)) + manifest = archive.read("META-INF/MANIFEST.MF").decode("utf-8") + if not any( + line.strip() in ("Build-Jdk-Spec: 1.8", "Build-Jdk-Spec: 8") + for line in manifest.splitlines() + ): + raise ValueError(f"{artifact} must be generated with JDK 8") + scripts = {name for name in names if name.endswith((".js", ".css"))} + if scripts != {"script.js", "stylesheet.css"} or any( + name.startswith(("legal/", "resources/")) for name in names + ): + raise ValueError(f"{artifact} contains unexpected Javadoc assets requiring license review") + print(f"Verified Java {'Javadoc' if javadoc else 'sources'} JAR: {artifact}") + + def cargo_metadata(target): return json.loads( subprocess.check_output( @@ -261,6 +301,8 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wheel", nargs="+", default=[], help="Wheel path or glob") parser.add_argument("--jar", nargs="+", default=[], help="JAR path or glob") + parser.add_argument("--sources-jar", nargs="+", default=[], help="Java sources JAR path or glob") + parser.add_argument("--javadoc-jar", nargs="+", default=[], help="JDK 8 Javadoc JAR path or glob") parser.add_argument( "--jar-native", action="append", @@ -277,8 +319,8 @@ def main(): ) args = parser.parse_args() - if not args.wheel and not args.jar and not args.source: - parser.error("at least one --source, --wheel, or --jar is required") + if not any((args.wheel, args.jar, args.source, args.sources_jar, args.javadoc_jar)): + parser.error("at least one artifact path or --source is required") try: if args.source: @@ -287,11 +329,16 @@ def main(): verify_wheel(artifact) for artifact in resolve_artifacts(args.jar): verify_jar(artifact, args.jar_native or JAR_NATIVE_FILES) + for artifact in resolve_artifacts(args.sources_jar): + verify_java_auxiliary(artifact) + for artifact in resolve_artifacts(args.javadoc_jar): + verify_java_auxiliary(artifact, javadoc=True) except ( KeyError, OSError, subprocess.CalledProcessError, ValueError, + ET.ParseError, zipfile.BadZipFile, ) as error: print(f"Binary artifact verification failed: {error}", file=sys.stderr)
