This is an automated email from the ASF dual-hosted git repository. yikun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push: new d58e178 [SPARK-41269][INFRA] Move image matrix into version's workflow d58e178 is described below commit d58e17890f07b4c8c8d212775a53c48dc3a6ce42 Author: Yikun Jiang <yikunk...@gmail.com> AuthorDate: Mon Nov 28 09:36:54 2022 +0800 [SPARK-41269][INFRA] Move image matrix into version's workflow ### What changes were proposed in this pull request? This patch refactors main workflow: - Move image matrix into version's workflow to make the main workflow more clear. And also will help downstream repo to only validate specified image type. - Move build steps into a same section ### Why are the changes needed? This will help downstream repo to only validate specified image type. After this patch, we will add a test to reuse spark docker workflow like: https://github.com/yikun/spark-docker/commit/45044cee2e8919de7e7353e74f8ca612ad16629a to help developers/users test their self build image. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI passed Closes #25 from Yikun/matrix-refactor. Authored-by: Yikun Jiang <yikunk...@gmail.com> Signed-off-by: Yikun Jiang <yikunk...@gmail.com> --- .github/workflows/build_3.3.0.yaml | 4 ++ .github/workflows/build_3.3.1.yaml | 4 ++ .github/workflows/main.yml | 76 ++++++++++++++++++++------------------ .github/workflows/publish.yml | 2 + 4 files changed, 51 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build_3.3.0.yaml b/.github/workflows/build_3.3.0.yaml index 7e7ce39..a4f8224 100644 --- a/.github/workflows/build_3.3.0.yaml +++ b/.github/workflows/build_3.3.0.yaml @@ -30,6 +30,9 @@ on: jobs: run-build: + strategy: + matrix: + image-type: ["all", "python", "scala", "r"] name: Run secrets: inherit uses: ./.github/workflows/main.yml @@ -37,3 +40,4 @@ jobs: spark: 3.3.0 scala: 2.12 java: 11 + image-type: ${{ matrix.image-type }} diff --git a/.github/workflows/build_3.3.1.yaml b/.github/workflows/build_3.3.1.yaml index f6a4b7d..9e5c082 100644 --- a/.github/workflows/build_3.3.1.yaml +++ b/.github/workflows/build_3.3.1.yaml @@ -30,6 +30,9 @@ on: jobs: run-build: + strategy: + matrix: + image-type: ["all", "python", "scala", "r"] name: Run secrets: inherit uses: ./.github/workflows/main.yml @@ -37,3 +40,4 @@ jobs: spark: 3.3.1 scala: 2.12 java: 11 + image-type: ${{ matrix.image-type }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 024b853..ebafcdc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,6 +47,11 @@ on: required: false type: string default: ghcr.io/apache/spark-docker + image-type: + description: The image type of the image (all, python, scala, r). + required: false + type: string + default: python jobs: main: @@ -60,41 +65,33 @@ jobs: image: registry:2 ports: - 5000:5000 - strategy: - matrix: - spark_version: - - ${{ inputs.spark }} - scala_version: - - ${{ inputs.scala }} - java_version: - - ${{ inputs.java }} - image_suffix: [python3-ubuntu, ubuntu, r-ubuntu, python3-r-ubuntu] steps: - name: Checkout Spark Docker repository uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - # This required by local registry - driver-opts: network=host - - - name: Generate tags + - name: Prepare - Generate tags run: | - TAG=scala${{ matrix.scala_version }}-java${{ matrix.java_version }}-${{ matrix.image_suffix }} + case "${{ inputs.image-type }}" in + all) SUFFIX=python3-r-ubuntu + ;; + python) SUFFIX=python3-ubuntu + ;; + r) SUFFIX=r-ubuntu + ;; + scala) SUFFIX=ubuntu + ;; + esac + TAG=scala${{ inputs.scala }}-java${{ inputs.java }}-$SUFFIX REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') TEST_REPO=localhost:5000/$REPO_OWNER/spark-docker IMAGE_NAME=spark - IMAGE_PATH=${{ matrix.spark_version }}/$TAG - UNIQUE_IMAGE_TAG=${{ matrix.spark_version }}-$TAG + IMAGE_PATH=${{ inputs.spark }}/$TAG + UNIQUE_IMAGE_TAG=${{ inputs.spark }}-$TAG IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$UNIQUE_IMAGE_TAG PUBLISH_REPO=${{ inputs.repository }} - PUBLISH_IMAGE_URL=`tools/manifest.py tags -i ${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ matrix.spark_version }}/${TAG}` + PUBLISH_IMAGE_URL=`tools/manifest.py tags -i ${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ inputs.spark }}/${TAG}` # Unique image tag in each version: 3.3.0-scala2.12-java11-python3-ubuntu echo "UNIQUE_IMAGE_TAG=${UNIQUE_IMAGE_TAG}" >> $GITHUB_ENV @@ -110,7 +107,7 @@ jobs: echo "PUBLISH_REPO=${PUBLISH_REPO}" >> $GITHUB_ENV echo "PUBLISH_IMAGE_URL=${PUBLISH_IMAGE_URL}" >> $GITHUB_ENV - - name: Print Image tags + - name: Prepare - Print Image tags run: | echo "UNIQUE_IMAGE_TAG: "${UNIQUE_IMAGE_TAG} echo "TEST_REPO: "${TEST_REPO} @@ -121,7 +118,16 @@ jobs: echo "PUBLISH_REPO:"${PUBLISH_REPO} echo "PUBLISH_IMAGE_URL:"${PUBLISH_IMAGE_URL} - - name: Build and push test image + - name: Build - Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Build - Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + # This required by local registry + driver-opts: network=host + + - name: Build - Build and push test image uses: docker/build-push-action@v3 with: context: ${{ env.IMAGE_PATH }} @@ -130,20 +136,20 @@ jobs: push: true - name : Test - Run spark application for standalone cluster on docker - run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ matrix.scala_version }} --spark-version ${{ matrix.spark_version }} + run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ inputs.scala }} --spark-version ${{ inputs.spark }} - name: Test - Checkout Spark repository uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark - ref: v${{ matrix.spark_version }} + ref: v${{ inputs.spark }} path: ${{ github.workspace }}/spark - name: Test - Cherry pick commits # Apache Spark enable resource limited k8s IT since v3.3.1, cherry-pick patches for old release # https://github.com/apache/spark/pull/36087#issuecomment-1251756266 - if: matrix.spark_version == '3.3.0' + if: inputs.spark == '3.3.0' working-directory: ${{ github.workspace }}/spark run: | # SPARK-38802: Add driverRequestCores/executorRequestCores supported @@ -159,7 +165,7 @@ jobs: # This is required after v2, now just keep same distribution with v1 # https://github.com/actions/setup-java/releases/tag/v2.0.0 distribution: 'zulu' - java-version: ${{ matrix.java_version }} + java-version: ${{ inputs.java }} - name: Test - Cache Scala, SBT and Maven uses: actions/cache@v3 @@ -169,13 +175,13 @@ jobs: build/scala-* build/*.jar ~/.sbt - key: build-${{ matrix.spark_version }}-scala${{ matrix.scala_version }}-java${{ matrix.java_version }} + key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }} - name: Test - Cache Coursier local repository uses: actions/cache@v3 with: path: ~/.cache/coursier - key: build-${{ matrix.spark_version }}-scala${{ matrix.scala_version }}-java${{ matrix.java_version }}-coursier + key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }}-coursier - name: Test - Start minikube run: | @@ -205,11 +211,11 @@ jobs: OPTS+="-Dspark.kubernetes.test.pythonImage=${IMAGE_NAME} " OPTS+="-Dspark.kubernetes.test.rImage=${IMAGE_NAME} " - if echo ${{ matrix.image_suffix }} | grep -q "python3-r-ubuntu"; then + if [ "${{ inputs.image-type }}" = "all" ]; then # Prepare test jar for client tests CONTAINER_TMP_NAME=spark-example-image docker create -ti --name $CONTAINER_TMP_NAME ${{ env.IMAGE_URL }} bash - docker cp $CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{ matrix.scala_version }}-${{ matrix.spark_version }}.jar . + docker cp $CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{ inputs.scala }}-${{ inputs.spark }}.jar . docker rm -f $CONTAINER_TMP_NAME # Prepare PV test PVC_TMP_DIR=$(mktemp -d) @@ -223,12 +229,12 @@ jobs: build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run SparkPi"' # Run basic test for PySpark image - if echo ${{ matrix.image_suffix }} | grep -q "python"; then + if [ "${{ inputs.image-type }}" = "python" ]; then build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run PySpark"' fi # Run basic test for SparkR image - if echo ${{ matrix.image_suffix }} | grep -q "r-"; then + if [ "${{ inputs.image-type }}" = "r" ]; then OPTS+="-Psparkr -Dtest.include.tags=r " build/sbt $OPTS 'kubernetes-integration-tests/testOnly' fi diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a44153b..4a07f5d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,6 +53,7 @@ jobs: matrix: scala: [2.12] java: [11] + image-type: ["all", "python", "scala", "r"] permissions: packages: write name: Run @@ -64,3 +65,4 @@ jobs: java: ${{ matrix.java }} publish: ${{ inputs.publish }} repository: ${{ inputs.repository }} + image-type: ${{ matrix.image-type }} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org