This is an automated email from the ASF dual-hosted git repository. xtsong pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-agents.git
commit 53d29e6fcf418c7625a04466cc6c4cc847ac3262 Author: sxnan <[email protected]> AuthorDate: Tue Oct 7 20:39:01 2025 +0800 [ci] Add release scripts --- .github/workflows/build_wheel.yml | 156 +++++++++++++++++++++++++++++++ .gitignore | 1 + pom.xml | 145 ++++++++++++++++++++++++++++ tools/build.sh | 3 +- tools/releasing/create_binary_release.sh | 111 ++++++++++++++++++++++ tools/releasing/create_release_branch.sh | 76 +++++++++++++++ tools/releasing/create_source_release.sh | 82 ++++++++++++++++ tools/releasing/deploy_staging_jars.sh | 46 +++++++++ 8 files changed, 619 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml new file mode 100644 index 0000000..6b9d153 --- /dev/null +++ b/.github/workflows/build_wheel.yml @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Wheel + +on: + workflow_dispatch: + +jobs: + lint: + name: Code Style Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check License Header + run: tools/check-license.sh + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'adopt' + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + - name: Check code style + run: ./tools/lint.sh -c + + java_tests: + name: java tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'macos-latest', 'ubuntu-latest' ] + steps: + - uses: actions/checkout@v4 + - name: Install java + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'adopt' + - name: Run Java Tests + run: tools/ut.sh -j + + python_tests: + name: python tests on ${{ matrix.os }} ${{ matrix.python-version}} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'macos-latest', 'ubuntu-latest' ] + python-version: [ '3.10', '3.11' ] + steps: + - uses: actions/checkout@v4 + - name: Install java + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'adopt' + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + - name: Run Python Tests + run: tools/ut.sh -p + + e2e_tests: + name: e2e tests on ${{ matrix.os }} ${{ matrix.python-version}} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'macos-latest', 'ubuntu-latest' ] + python-version: [ '3.10', '3.11' ] + steps: + - uses: actions/checkout@v4 + - name: Install java + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'adopt' + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + - name: Build flink-agents + run: bash tools/build.sh + - name: Run e2e tests + run: bash tools/e2e.sh + + build_python_wheels: + name: "Build Python Wheels" + runs-on: ubuntu-latest + steps: + - name: "Checkout the repository" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install java + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'adopt' + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + - name: "Build python wheels" + run: bash tools/build.sh + env: + # Skip -musllinux on Linux builds + CIBW_SKIP: "*-musllinux*" + # Use manylinux2014 on Linux + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_BEFORE_ALL_LINUX: pip install patchelf + CIBW_BEFORE_BUILD_LINUX: pip install --upgrade pip && pip install --group python/pyproject.toml:dev + CIBW_ENVIRONMENT_LINUX: CFLAGS="-I. -include ./dev/glibc_version_fix.h" + # Run auditwheel repair on Linux + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair -w {dest_dir} {wheel}" + - name: "Upload python wheels" + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ github.run_number }} + path: python/dist/** diff --git a/.gitignore b/.gitignore index c34bd50..345b3c2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ python/uv.lock python/.DS_Store __pycache__ **/dependency-reduced-pom.xml +/tools/releasing/release/ diff --git a/pom.xml b/pom.xml index b4bf41a..d78e2a6 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,9 @@ under the License. <slf4j.version>1.7.36</slf4j.version> <assertj.version>3.23.1</assertj.version> <mockito.version>5.8.0</mockito.version> + + <gpg.useagent>true</gpg.useagent> + <arguments /> </properties> <modules> @@ -84,7 +87,149 @@ under the License. </dependency> </dependencies> + <profiles> + <profile> + <id>release</id> + <activation> + <property> + <name>release</name> + </property> + </activation> + <properties> + <target.java.version>11</target.java.version> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-gpg-plugin</artifactId> + <version>1.4</version> + <executions> + <execution> + <id>sign-artifacts</id> + <phase>verify</phase> + <goals> + <goal>sign</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-maven</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <!-- versions for certain build tools are enforced to match the CI setup --> + <!-- the rules below should stay in sync with Flink Release wiki documentation and the CI scripts --> + <requireJavaVersion> + <version>[${target.java.version}.*,)</version> + </requireJavaVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <id>attach-javadocs</id> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-release-plugin</artifactId> + <version>2.1</version> + <configuration> + <mavenExecutorId>forked-path</mavenExecutorId> + <useReleaseProfile>false</useReleaseProfile> + <arguments>${arguments} -Psonatype-oss-release</arguments> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + </profile> + + <profile> + <!-- used for SNAPSHOT and regular releases --> + <id>docs-and-source</id> + <activation> + <property> + <name>docs-and-source</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>3.2.1</version> + <executions> + <execution> + <id>attach-sources</id> + <goals> + <goal>jar-no-fork</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <configuration> + <quiet>true</quiet> + </configuration> + <executions> + <execution> + <id>attach-javadocs</id> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <version>3.8.0</version> + <configuration> + <quiet>true</quiet> + <detectOfflineLinks>false</detectOfflineLinks> + <release>11</release> + <additionalJOptions> + <additionalJOption>-Xdoclint:none</additionalJOption> + <additionalJOption>--allow-script-in-comments</additionalJOption> + <!-- Suppress the error that is accepted by JDK 8 but not by JDK 11. --> + <additionalJOption>--ignore-source-errors</additionalJOption> + </additionalJOptions> + </configuration> + </plugin> + </plugins> + </pluginManagement> + <plugins> <plugin> <groupId>com.diffplug.spotless</groupId> diff --git a/tools/build.sh b/tools/build.sh index ab35cad..baab628 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -37,7 +37,8 @@ PROJECT_ROOT="${BASE_DIR}/../" # build java if $build_java; then - mvn clean package -DskipTests + mvn --version + mvn clean package -DskipTests -B fi # copy flink-agents-dist jar to python lib diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh new file mode 100755 index 0000000..41e873d --- /dev/null +++ b/tools/releasing/create_binary_release.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +SKIP_GPG=${SKIP_GPG:-false} +MVN=${MVN:-mvn} + +if [ -z "${RELEASE_VERSION:-}" ]; then + echo "RELEASE_VERSION was not set." + exit 1 +fi + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +if [ "$(uname)" == "Darwin" ]; then + SHASUM="shasum -a 512" +else + SHASUM="sha512sum" +fi + +cd .. + +FLINK_AGENTS_DIR=`pwd` +RELEASE_DIR=${FLINK_AGENTS_DIR}/tools/releasing/release +PYTHON_RELEASE_DIR=${RELEASE_DIR}/python +mkdir -p ${PYTHON_RELEASE_DIR} + +########################### + +# build maven package, create Flink distribution, generate signature +make_binary_release() { + + echo "Creating binary release" + + DIST_JAR_NAME="flink-agents-dist-${RELEASE_VERSION}.jar" + + # enable release profile here (to check for the maven version) + $MVN clean package -Prelease -pl dist -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests + + cd dist/target + cp $DIST_JAR_NAME ${RELEASE_DIR} + cd ${RELEASE_DIR} + + # Sign sha the tgz + if [ "$SKIP_GPG" == "false" ] ; then + gpg --armor --detach-sig "${DIST_JAR_NAME}" + fi + $SHASUM "${DIST_JAR_NAME}" > "${DIST_JAR_NAME}.sha512" + + cd ${FLINK_AGENTS_DIR} +} + +make_python_release() { + FLINK_AGENTS_VERSION=${RELEASE_VERSION/-SNAPSHOT/.dev0} + cd python/dist + + # Need to move the downloaded wheel packages from Azure CI to the directory flink-python/dist manually. + for wheel_file in *.whl; do + if [[ ! ${wheel_file} =~ ^flink_agents-$FLINK_AGENTS_VERSION- ]]; then + echo -e "\033[31;1mThe file name of the python package: ${wheel_file} is not consistent with given release version: ${FLINK_AGENTS_VERSION}!\033[0m" + exit 1 + fi + cp ${wheel_file} "${PYTHON_RELEASE_DIR}/${wheel_file}" + done + + cd ${PYTHON_RELEASE_DIR} + + # Sign sha the tgz and wheel packages + if [ "$SKIP_GPG" == "false" ] ; then + for wheel_file in *.whl; do + gpg --armor --detach-sig "${wheel_file}" + done + fi + + for wheel_file in *.whl; do + $SHASUM "${wheel_file}" > "${wheel_file}.sha512" + done + + cd ${FLINK_AGENTS_DIR} +} + +make_binary_release +make_python_release diff --git a/tools/releasing/create_release_branch.sh b/tools/releasing/create_release_branch.sh new file mode 100755 index 0000000..ebe7683 --- /dev/null +++ b/tools/releasing/create_release_branch.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +RELEASE_CANDIDATE=${RELEASE_CANDIDATE:-none} +MVN=${MVN:-mvn} + +if [ -z "${NEW_VERSION:-}" ]; then + echo "NEW_VERSION was not set." + exit 1 +fi + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +########################### + +cd .. + +target_branch=release-$NEW_VERSION +if [ "$RELEASE_CANDIDATE" != "none" ]; then + target_branch=$target_branch-rc$RELEASE_CANDIDATE +fi + +git checkout -b $target_branch + +#change version in all pom files +$MVN org.codehaus.mojo:versions-maven-plugin:2.8.1:set -DnewVersion=$NEW_VERSION -DgenerateBackupPoms=false + +#change version of documentation +cd docs +perl -pi -e "s#^ Version = .*# Version = \"${NEW_VERSION}\"#" config.toml + +# The version in the title should not contain the bugfix version (e.g. 1.3) +VERSION_TITLE=$(echo $NEW_VERSION | sed 's/\.[^.]*$//') +perl -pi -e "s#^ VersionTitle = .*# VersionTitle = \"${VERSION_TITLE}\"#" config.toml +cd .. + +#change version of flink-agents +cd python +perl -pi -e "s#^version = \".*\"#version = \"${NEW_VERSION}\"#" pyproject.toml +cd .. + +git commit -am "Commit for release $NEW_VERSION" + +RELEASE_HASH=`git rev-parse HEAD` +echo "Echo created release hash $RELEASE_HASH" + +echo "Done. Don't forget to create the signed release tag at the end and push the changes." diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh new file mode 100755 index 0000000..b64519f --- /dev/null +++ b/tools/releasing/create_source_release.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +MVN=${MVN:-mvn} + +if [ -z "${RELEASE_VERSION:-}" ]; then + echo "RELEASE_VERSION was not set." + exit 1 +fi + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +if [ "$(uname)" == "Darwin" ]; then + SHASUM="shasum -a 512" +else + SHASUM="sha512sum" +fi + +########################### + +cd .. + +FLINK_AGENTS=`pwd` +RELEASE_DIR=${FLINK_AGENTS}/tools/releasing/release +CLONE_DIR=${RELEASE_DIR}/flink-tmp-clone + +echo "Creating source package" + +mkdir -p ${RELEASE_DIR} + +# create a temporary git clone to ensure that we have a pristine source release +git clone ${FLINK_AGENTS} ${CLONE_DIR} +cd ${CLONE_DIR} + +rsync -a \ + --exclude ".git" --exclude ".gitignore" --exclude ".gitattributes" --exclude "azure-pipelines.yml" --exclude ".asf.yaml" \ + --exclude "CHANGELOG" --exclude ".github" --exclude "target" \ + --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \ + --exclude "docs/public" --exclude "docs/resources" --exclude "docs/themes" \ + --exclude "*/dependency-reduced-pom.xml" \ + --exclude "python/lib" --exclude "python/build" \ + --exclude "python/dist" --exclude "python/flink_agents.egg-info" \ + --exclude "python/.tox" --exclude "python/.cache" --exclude "python/uv.lock" \ + --exclude "python/.pytest_cache" \ + . flink-agents-$RELEASE_VERSION + +tar czf ${RELEASE_DIR}/flink-agents-${RELEASE_VERSION}-src.tgz flink-agents-$RELEASE_VERSION +gpg --armor --detach-sig ${RELEASE_DIR}/flink-agents-$RELEASE_VERSION-src.tgz +cd ${RELEASE_DIR} +$SHASUM flink-agents-$RELEASE_VERSION-src.tgz > flink-agents-$RELEASE_VERSION-src.tgz.sha512 + +cd ${CURR_DIR} +rm -rf ${CLONE_DIR} diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh new file mode 100755 index 0000000..e076869 --- /dev/null +++ b/tools/releasing/deploy_staging_jars.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +MVN=${MVN:-mvn} +CUSTOM_OPTIONS=${CUSTOM_OPTIONS:-} + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +########################### + +cd .. + +echo "Deploying to repository.apache.org" + +COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10 $CUSTOM_OPTIONS" + +$MVN clean deploy $COMMON_OPTIONS
