This is an automated email from the ASF dual-hosted git repository. ajfabbri pushed a commit to branch af/hadoop-19877 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ac00cb2176e97f987adeeb039c2765b5888c5b10 Author: Aaron Fabbri <[email protected]> AuthorDate: Mon May 4 19:59:58 2026 -0700 HADOOP-19877: ci: initial S3A integration test CI --- .github/workflows/cloud_aws.yml | 32 +++++++++ .github/workflows/tmpl_cloud_aws.yml | 135 +++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/.github/workflows/cloud_aws.yml b/.github/workflows/cloud_aws.yml new file mode 100644 index 00000000000..88381b5d357 --- /dev/null +++ b/.github/workflows/cloud_aws.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Cloud - AWS integration tests" + +on: + pull_request: + paths: + - 'hadoop-tools/hadoop-aws/**' + +jobs: + run-aws-integration: + permissions: + packages: write + name: Run + uses: ./.github/workflows/tmpl_cloud_aws.yml diff --git a/.github/workflows/tmpl_cloud_aws.yml b/.github/workflows/tmpl_cloud_aws.yml new file mode 100644 index 00000000000..abb0789bf47 --- /dev/null +++ b/.github/workflows/tmpl_cloud_aws.yml @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: AWS Integration + +on: + workflow_call: + inputs: + java: + required: false + type: string + default: 17 + branch: + required: false + type: string + description: Branch to run the build against + default: trunk + os: + required: false + type: string + description: Operating system to run the build on + default: ubuntu_24 + +# Security: Default to minimal permissions for workflow. +permissions: {} + +concurrency: + group: >- + cloud-aws + ${{ github.workflow }} + ${{ github.repository == 'apache/hadoop' && github.run_id || github.ref }} + ${{ inputs.java }} + ${{ inputs.branch }} + ${{ inputs.os }} + cancel-in-progress: true + +env: + BUCKET_NAME: hadoop-ci + +jobs: + precondition: + runs-on: ubuntu-24.04 + outputs: + build_image_url: ${{ steps.img.outputs.build_image_url }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/build_image_url + id: img + with: + branch: ${{ inputs.branch }} + os: ${{ inputs.os }} + + build-image: + runs-on: ubuntu-24.04 + needs: [ precondition ] + permissions: + packages: write + outputs: + uid: ${{ steps.build_img.outputs.uid }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/build_image + id: build_img + with: + branch: ${{ inputs.branch }} + os: ${{ inputs.os }} + build_image_url: ${{ needs.precondition.outputs.build_image_url }} + + test: + name: S3A Integration Tests (Java ${{ inputs.java }}) + needs: [ precondition, build-image ] + runs-on: ubuntu-24.04 + container: + image: ${{ needs.precondition.outputs.build_image_url }} + options: --user ${{ needs.build-image.outputs.uid }} + steps: + - uses: actions/checkout@v6 + - name: Setup JDK ${{ inputs.java }} + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: ${{ inputs.java }} + cache: 'maven' + - name: Localstack setup + # Security: 0.3.2 is an immutable commit, see: + # https://github.com/localstack/setup-localstack/commit/7c8a0cb3405bc58be4c8f763f812aa000bc46303 + uses: LocalStack/[email protected] + with: + image-tag: 'latest' + install-awslocal: 'true' + env: + AWS_REGION: us-west-2 + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_CI_KEY }} + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + + - name: Maven Build + shell: bash + run: ./mvnw install -B -DskipTests -am -pl hadoop-tools/hadoop-aws + - name: Create localstack s3 bucket + shell: bash + run: awslocal s3 mb s3://${{ env.BUCKET_NAME }} + + - name: S3A Integration Tests + shell: bash + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_REGION: us-west-2 + run: | + ./mvnw verify -B -pl hadoop-tools/hadoop-aws \ + -DskipITs=false \ + -Dtest=none \ + -Dscale=true \ + -Dfs.s3a.access.key=test \ + -Dfs.s3a.secret.key=test \ + -Dfs.s3a.endpoint=http://localhost:4566 \ + -Dfs.s3a.connection.ssl.enabled=false \ + -Dfs.s3a.path.style.access=true \ + -Dfs.s3a.endpoint.region=us-west-2 \ + -Dtest.fs.s3a.name=s3a://${{ env.BUCKET_NAME }}/ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
