dalelane commented on code in PR #28831:
URL: https://github.com/apache/flink/pull/28831#discussion_r3665239029
##########
.github/workflows/nightly.yml:
##########
@@ -77,6 +109,163 @@ jobs:
s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+ snapshot_binary:
+ name: "Snapshot Binary Release"
+ runs-on: ubuntu-24.04
+ container:
+ image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy
+ options: --init --privileged
+ env:
+ MOUNTED_WORKING_DIR: /__w/flink/flink
+ CONTAINER_LOCAL_WORKING_DIR: /root/flink
+ MAVEN_REPO_FOLDER: /root/.m2/repository
+ MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository
+ RELEASE_DIR: /root/flink/tools/releasing/release
+ steps:
+ - name: "Flink Checkout"
+ uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - name: "Initialize job"
+ uses: "./.github/actions/job_init"
+ with:
+ jdk_version: 11
+ maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }}
+ source_directory: ${{ env.MOUNTED_WORKING_DIR }}
+ target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+ - name: "Build snapshot binary release"
+ working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+ shell: bash
+ run: |
+ # errexit needs to be disabled explicitly here because
maven-utils.sh handles the error if a mirror is not available
+ set +o errexit
+ source ./tools/ci/maven-utils.sh
+ set -o errexit
+ run_mvn -version
+ export MVN="run_mvn"
+ export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate
-Dexpression=project.version -q -DforceStdout)
+ echo "Determined RELEASE_VERSION as '$RELEASE_VERSION'"
+ cd tools
+ MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true
./releasing/create_binary_release.sh
+ echo "Created files:"
+ find ./releasing/release
+ cd ..
+ # Uploaded for every branch so that the binaries can be inspected and
+ # compared against the ones the Azure pipeline produces.
+ - name: "Upload snapshot binary release as build artifact"
+ uses: actions/upload-artifact@v7
+ with:
+ name: snapshot-binary-release-${{ github.run_number }}
+ path: ${{ env.RELEASE_DIR }}
+ if-no-files-found: error
+ retention-days: 5
+ - name: "Upload artifacts to S3"
+ if: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES),
github.ref_name) }}
+ working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+ shell: bash
+ run: |
+ source ./tools/ci/deploy_nightly_to_s3.sh
+ # branches in the shadow phase upload to a GHA-specific prefix, so
that
+ # they don't overwrite the builds AZP is still uploading
+ upload_to_s3 ./tools/releasing/release "${S3_PREFIX}"
+ env:
+ S3_PREFIX: ${{ contains(fromJSON(env.SHADOW_SNAPSHOT_BRANCHES),
github.ref_name) && 'gha-trial/' || '' }}
+ ARTIFACTS_S3_BUCKET: ${{ secrets.ARTIFACTS_S3_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }}
+
+ snapshot_maven:
+ name: "Snapshot Maven Deploy"
+ runs-on: ubuntu-24.04
+ timeout-minutes: 240
+ container:
+ image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy
+ options: --init --privileged
+ env:
+ MOUNTED_WORKING_DIR: /__w/flink/flink
+ CONTAINER_LOCAL_WORKING_DIR: /root/flink
+ MAVEN_REPO_FOLDER: /root/.m2/repository
+ MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository
+ STAGING_JARS_DIR: /root/staging-jars
+ STAGING_JARS_TARBALL: /root/staging-jars.tar.gz
+ steps:
+ - name: "Flink Checkout"
+ uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - name: "Initialize job"
+ uses: "./.github/actions/job_init"
+ with:
+ jdk_version: 11
+ maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }}
+ source_directory: ${{ env.MOUNTED_WORKING_DIR }}
+ target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+ - name: "Deploy Maven snapshot"
+ working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+ shell: bash
+ run: |
+ # errexit needs to be disabled explicitly here because
maven-utils.sh handles the error if a mirror is not available
+ set +o errexit
+ source ./tools/ci/maven-utils.sh
+ set -o errexit
+ run_mvn -version
+
+ cd tools
+ cat << EOF > deploy-settings.xml
+ <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+
https://maven.apache.org/xsd/settings-1.0.0.xsd";>
+ <servers>
+ <server>
+ <id>apache.snapshots.https</id>
+ <username>${MAVEN_DEPLOY_USER}</username>
+ <password>${MAVEN_DEPLOY_PASS}</password>
+ </server>
+ </servers>
+ <mirrors>
+ <mirror>
+ <id>google-maven-central</id>
+ <name>GCS Maven Central mirror</name>
+
<url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
+ <mirrorOf>central</mirrorOf>
+ </mirror>
+ </mirrors>
+ </settings>
+ EOF
+
+ export CUSTOM_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR}
-Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml"
+
+ if [ "${PUBLISH_SNAPSHOTS}" != "true" ]; then
+ # this branch's snapshot jars are published by the Azure pipeline,
so
+ # deploy into a local repository that is kept as a build artifact
+ # instead of pushing to repository.apache.org
+ echo "'${GITHUB_REF_NAME}' does not publish snapshots - deploying
to ${STAGING_JARS_DIR} instead of repository.apache.org"
+ export CUSTOM_OPTIONS="${CUSTOM_OPTIONS}
-DaltDeploymentRepository=staging::file://${STAGING_JARS_DIR}"
+ fi
+
+ export MVN_RUN_VERBOSE=true
+ ./releasing/deploy_staging_jars.sh
+ env:
+ MAVEN_DEPLOY_USER: ${{ secrets.MAVEN_DEPLOY_USER }}
+ MAVEN_DEPLOY_PASS: ${{ secrets.MAVEN_DEPLOY_PASS }}
+ PUBLISH_SNAPSHOTS: ${{
contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }}
+ - name: "Archive locally deployed snapshot jars"
+ if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES),
github.ref_name) }}
+ shell: bash
+ run: |
+ du -sh "${STAGING_JARS_DIR}"
+ tar --create --gzip --file "${STAGING_JARS_TARBALL}" -C
"${STAGING_JARS_DIR}" .
+ ls -lh "${STAGING_JARS_TARBALL}"
+ - name: "Upload snapshot jars as build artifact"
+ if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES),
github.ref_name) }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: snapshot-maven-jars-${{ github.run_number }}
+ path: ${{ env.STAGING_JARS_TARBALL }}
+ if-no-files-found: error
+ retention-days: 5
Review Comment:
re: https://github.com/dalelane/flink/pull/1#discussion_r3636069514
> I'm wondering whether it's feasible to have all the generated jars in a
tar.gz build artifact or whether we hit some size limits here.
@XComp setting a retention time is a nod towards your comment on the storage
requirements here. We could make this even shorter, I largely picked 5 days out
of the air.
FWIW gzip'ping doesn't seem to make much difference.
I tried building a sample of modules - which suggests that the total size
will be in the 1gb - 1.5gb ballpark. That is bigger than the snapshot binaries
(which seem to be in the 600mb ballpark)
If this is too big, we could just tar the primary jars, and drop the
sources/javadoc versions? Is that worth doing?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
