This is an automated email from the ASF dual-hosted git repository.
ajfabbri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6337d964188 HADOOP-19893: S3A CI tests for fork PRs (#8530)
6337d964188 is described below
commit 6337d9641880610c4d1bddba17da2e9f0484a00d
Author: Aaron Fabbri <[email protected]>
AuthorDate: Fri Jul 24 15:25:39 2026 -0700
HADOOP-19893: S3A CI tests for fork PRs (#8530)
* HADOOP-19893. Add manual cloud_aws CI trigger for fork PRs
- Add workflow_dispatch inputs in cloud_aws.yml to run CI by fork PR number
- Resolve PR head repo/SHA via API and run tests against that exact fork
commit.
- Keep automatic cloud_aws CI runs only for same-repo pull_request events.
- tmpl_cloud_aws.yml adds checkout_repository/checkout_ref inputs.
- Pass LOCALSTACK_CI_KEY explicitly to reusable workflow (required).
- Post S3A Test Summary commit status for manual runs when CI wasn't
skipped.
- Stash PR number and branch repo details for reporting workflow.
- Tweak report_cloud_aws.yml to consume PR details, using stashed target
commit
for report.
- Add sticky comment hint to S3A PRs from forks
- Clarify status of manual trigger notification
- ci: update docker/login-action uses
The existing version SHA did not match the ASF Infra's action.yml whitelist,
causing CI to fail to start.
---
.github/actions/build_image/action.yml | 2 +-
.../gha-tests/hadoop-aws-localstack-excludes.txt | 7 ++
.github/workflows/cloud_aws.yml | 101 ++++++++++++++++-
.github/workflows/notify_cloud_aws.yml | 120 +++++++++++++++++++++
.github/workflows/report_cloud_aws.yml | 37 ++++++-
.github/workflows/tmpl_build_and_test.yml | 2 +-
.github/workflows/tmpl_build_image_cache.yml | 2 +-
.github/workflows/tmpl_cloud_aws.yml | 26 ++++-
8 files changed, 287 insertions(+), 10 deletions(-)
diff --git a/.github/actions/build_image/action.yml
b/.github/actions/build_image/action.yml
index 9027c1d02d3..810813e9aec 100644
--- a/.github/actions/build_image/action.yml
+++ b/.github/actions/build_image/action.yml
@@ -37,7 +37,7 @@ runs:
using: composite
steps:
- name: Login to GitHub Container Registry
- uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 #
v4.4.0
+ uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c #
v4.5.0
with:
registry: ghcr.io
username: ${{ github.actor }}
diff --git a/.github/gha-tests/hadoop-aws-localstack-excludes.txt
b/.github/gha-tests/hadoop-aws-localstack-excludes.txt
index d16ff8957a4..3e4fa234e11 100644
--- a/.github/gha-tests/hadoop-aws-localstack-excludes.txt
+++ b/.github/gha-tests/hadoop-aws-localstack-excludes.txt
@@ -23,6 +23,8 @@
# TODO see if we can enable any of these...
# tests that depend on public S3 buckets
+# TODO may be able disable these with s3a test config (see testing.md under
+# third party)
**/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
**/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
**/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
@@ -48,6 +50,7 @@
# expected: <true> but was: <false>
# 2. testUpdateDeepDirectoryStructureToRemote():
# AssertionFailedError: Files Copied value 2 above maximum 1 ==> expected:
<true> but was: <false>
+# TODO see if this is a timestamp issue
**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
# A number of failures with vectored read tests
@@ -59,6 +62,8 @@
# s3a://hadoop-ci/job-00/test:
# software.amazon.awssdk.services.s3.model.S3Exception: The AWS Access Key Id
you
# provided does not exist in our records. (Service: S3, Status Code: 403
+# TODO see if tests are still connecting to AWS (validate which endpoint is
+# being used)
**/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
**/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -76,4 +81,6 @@
# testSeeksWithLruEviction java.util.concurrent.TimeoutException: timed out
# after 180 seconds
+# TODO consider removing this feature & test. This impl. is problematic,
+# vectored IO addresses some of this, and the Amazon analytics input stream
addresses the rest of it.
**/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java
diff --git a/.github/workflows/cloud_aws.yml b/.github/workflows/cloud_aws.yml
index 7d48413fb0d..c33690887dc 100644
--- a/.github/workflows/cloud_aws.yml
+++ b/.github/workflows/cloud_aws.yml
@@ -20,6 +20,7 @@
name: "Cloud-AWS"
on:
+ # pull requests will automatically trigger S3A tests for non-forked PRs
pull_request:
paths:
- 'hadoop-tools/hadoop-aws/**'
@@ -27,8 +28,88 @@ on:
- '.github/actions/build_image**'
- '.github/gha-tests/hadoop-aws*excludes.txt'
+ # For fork PRs, S3A integration tests must be manually triggered.
+ # Although our auth. key for localstack is not a very sensitive secret,
+ # we don't want to leak it to PRs that we haven't reviewed yet.
+ workflow_dispatch:
+ inputs:
+ pr_number:
+ description: Pull request number from a fork repository
+ required: true
+ type: string
+ java:
+ description: Java version for the Cloud-AWS run
+ required: false
+ type: string
+ default: '25'
+ os:
+ description: OS image key for the test container
+ required: false
+ type: string
+ default: ubuntu_24
+ runner_os:
+ description: Runner label used to execute jobs
+ required: false
+ type: string
+ default: ubuntu-24.04
+
jobs:
+ # Stash PR's repository and commit, so we can properly add a test summary
+ # in both the forked-repo, and non-forked case.
+ resolve-fork-pr:
+ if: github.event_name == 'workflow_dispatch'
+ runs-on: ubuntu-slim
+ permissions:
+ pull-requests: read
+ outputs:
+ head_repo: ${{ steps.pr.outputs.head_repo }}
+ head_sha: ${{ steps.pr.outputs.head_sha }}
+ steps:
+ - name: Resolve PR head SHA and repository
+ id: pr
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #
v9
+ with:
+ script: |
+ const prNumber = Number('${{ inputs.pr_number }}');
+ if (!Number.isInteger(prNumber) || prNumber <= 0) {
+ throw new Error(`Invalid pull request number: '${{
inputs.pr_number }}'`);
+ }
+
+ const { data: pr } = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: prNumber,
+ });
+
+ if (!pr.head.repo.fork) {
+ throw new Error(`PR #${prNumber} is not from a fork repository.
Use the pull_request-triggered Cloud-AWS workflow instead.`);
+ }
+
+ core.setOutput('head_repo', pr.head.repo.full_name);
+ core.setOutput('head_sha', pr.head.sha);
+
+ console.log(`Resolved PR #${prNumber}:
${pr.head.repo.full_name}@${pr.head.sha}`);
+
+ - name: Write manual target metadata
+ shell: bash
+ run: |
+ cat > manual-target.json <<EOF
+ {
+ "pr_number": "${{ inputs.pr_number }}",
+ "head_repo": "${{ steps.pr.outputs.head_repo }}",
+ "head_sha": "${{ steps.pr.outputs.head_sha }}"
+ }
+ EOF
+
+ - name: Upload manual target metadata
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
# v7.0.1
+ with:
+ name: cloud-aws-manual-target
+ path: manual-target.json
+ retention-days: 7
+
run-aws-integration:
+ if: github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.full_name == github.repository
# Security: write privileges are needed to update PR status and upload
test results.
# Package write is for building toolchain container images on demand, but
ghcr.io access is
# scoped to the repository the actions run on.
@@ -37,8 +118,26 @@ jobs:
contents: read
name: Run
uses: ./.github/workflows/tmpl_cloud_aws.yml
- secrets: inherit # inherit LOCALSTACK_CI_KEY
+ secrets:
+ LOCALSTACK_CI_KEY: ${{ secrets.LOCALSTACK_CI_KEY }}
with:
java: 25
os: ubuntu_24
runner_os: ubuntu-24.04
+
+ run-aws-integration-manual:
+ if: github.event_name == 'workflow_dispatch'
+ needs: [ resolve-fork-pr ]
+ permissions:
+ packages: write
+ contents: read
+ name: Run (manual fork PR)
+ uses: ./.github/workflows/tmpl_cloud_aws.yml
+ secrets:
+ LOCALSTACK_CI_KEY: ${{ secrets.LOCALSTACK_CI_KEY }}
+ with:
+ java: ${{ inputs.java }}
+ os: ${{ inputs.os }}
+ runner_os: ${{ inputs.runner_os }}
+ checkout_repository: ${{ needs.resolve-fork-pr.outputs.head_repo }}
+ checkout_ref: ${{ needs.resolve-fork-pr.outputs.head_sha }}
diff --git a/.github/workflows/notify_cloud_aws.yml
b/.github/workflows/notify_cloud_aws.yml
new file mode 100644
index 00000000000..d9e9140a98b
--- /dev/null
+++ b/.github/workflows/notify_cloud_aws.yml
@@ -0,0 +1,120 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Add a sticky comment to hadoop-aws PRs from forked repos with a hint that
+# integration tests must be manually triggered by a maintainer.
+#
+name: "Cloud-AWS PR Update"
+
+# Security: This privileged workflow uses pull_request_target but does not
+# check out or execute untrusted code. It only creates a check run and a PR
+# comment in the base repository.
+on:
+ pull_request_target:
+ types: [opened, reopened, synchronize]
+ paths:
+ - 'hadoop-tools/hadoop-aws/**'
+ - '.github/workflows/*cloud_aws.yml'
+ - '.github/actions/build_image**'
+ - '.github/gha-tests/hadoop-aws*excludes.txt'
+
+jobs:
+ notify:
+ if: github.event.pull_request.head.repo.full_name != github.repository
+ name: "Notify Cloud-AWS"
+ runs-on: ubuntu-slim
+ permissions:
+ checks: write
+ pull-requests: write
+ steps:
+ - name: Post approval-required check and sticky comment
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #
v9
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const marker = '<!-- cloud-aws-fork-pr-note -->';
+ const pr = context.payload.pull_request;
+ const workflowUrl =
`https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/cloud_aws.yml`;
+ const dispatchCommand = `gh workflow run cloud_aws.yml -R
${context.repo.owner}/${context.repo.repo} -f pr_number=${pr.number}`;
+
+ const checkTitle = 'Cloud-AWS manual trigger instructions';
+ const checkSummary = [
+ 'S3A tests must be manually triggered for fork pull requests.',
+ 'A maintainer should:\n',
+ '- Confirm the PR does not contain risky changes to actions in',
+ ' .github/workflows and .github/actions.\n',
+ '- *Approve workflows* for this fork PR if prompted.\n',
+ `- Run [Cloud-AWS workflow](${workflowUrl}) with
\`pr_number=${pr.number}\``,
+ ` (or via CLI: \`${dispatchCommand}\`)\n`,
+ '\n',
+ `fork head: \`${pr.head.repo.full_name}@${pr.head.sha.slice(0,
12)}\``,
+ ].join(' ');
+
+ await github.rest.checks.create({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: 'Cloud-AWS / Manual Trigger Info',
+ head_sha: pr.head.sha,
+ status: 'completed',
+ conclusion: 'neutral',
+ output: {
+ title: checkTitle,
+ summary: checkSummary,
+ },
+ });
+
+ const commentBody = [
+ marker,
+ `ℹ️ **${checkTitle}**: ${checkSummary}`,
+ ].join('\n');
+
+ try {
+ const comments = await
github.paginate(github.rest.issues.listComments, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: pr.number,
+ per_page: 100,
+ });
+
+ const existing = comments.find(comment => comment.body &&
comment.body.includes(marker));
+ if (existing) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: existing.id,
+ body: commentBody,
+ });
+ core.info(`Updated sticky comment ${existing.id}`);
+ } else {
+ const created = await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: pr.number,
+ body: commentBody,
+ });
+ core.info(`Created sticky comment ${created.data.id}`);
+ }
+ } catch (error) {
+ if (error.status === 403) {
+ core.warning('Unable to write sticky PR comment due to repo
token policy.');
+ core.warning(`GitHub API message: ${error.message}`);
+ } else {
+ throw error;
+ }
+ }
diff --git a/.github/workflows/report_cloud_aws.yml
b/.github/workflows/report_cloud_aws.yml
index 5ea9037dda3..75a78711b2e 100644
--- a/.github/workflows/report_cloud_aws.yml
+++ b/.github/workflows/report_cloud_aws.yml
@@ -32,8 +32,39 @@ permissions:
jobs:
report:
runs-on: ubuntu-latest
- if: github.event.workflow_run.event == 'pull_request'
+ if: (github.event.workflow_run.event == 'pull_request' ||
github.event.workflow_run.event == 'workflow_dispatch') &&
github.event.workflow_run.conclusion != 'skipped'
steps:
+ - name: Download manual target metadata
+ if: github.event.workflow_run.event == 'workflow_dispatch'
+ uses:
actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: cloud-aws-manual-target
+ run-id: ${{ github.event.workflow_run.id }}
+ path: manual-target
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Resolve status target SHA
+ id: target_sha
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #
v9
+ env:
+ WORKFLOW_EVENT: ${{ github.event.workflow_run.event }}
+ WORKFLOW_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+ with:
+ script: |
+ const fs = require('node:fs');
+
+ let targetSha = process.env.WORKFLOW_HEAD_SHA;
+ if (process.env.WORKFLOW_EVENT === 'workflow_dispatch') {
+ const payload =
JSON.parse(fs.readFileSync('manual-target/manual-target.json', 'utf8'));
+ if (!payload.head_sha) {
+ throw new Error('manual-target.json is missing head_sha');
+ }
+ targetSha = payload.head_sha;
+ }
+
+ core.setOutput('value', targetSha);
+ console.log(`Status target SHA: ${targetSha}`);
+
- name: S3A Test Report
id: report
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 #
v3.0.0
@@ -53,7 +84,7 @@ jobs:
fail-on-error: 'false'
- name: Post S3A commit status
- uses: actions/github-script@v7
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #
v9
if: always()
with:
script: |
@@ -78,7 +109,7 @@ jobs:
const params = {
...context.repo,
- sha: context.payload.workflow_run.head_sha,
+ sha: '${{ steps.target_sha.outputs.value }}',
state,
description,
context: 'S3A Test Summary',
diff --git a/.github/workflows/tmpl_build_and_test.yml
b/.github/workflows/tmpl_build_and_test.yml
index ac04faa78df..e5beb7cfa41 100644
--- a/.github/workflows/tmpl_build_and_test.yml
+++ b/.github/workflows/tmpl_build_and_test.yml
@@ -118,7 +118,7 @@ jobs:
with:
fetch-depth: 0
- name: Login to GitHub Container Registry
- uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 #
v4.4.0
+ uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c #
v4.5.0
with:
registry: ghcr.io
username: ${{ github.actor }}
diff --git a/.github/workflows/tmpl_build_image_cache.yml
b/.github/workflows/tmpl_build_image_cache.yml
index ad8c792c957..0e609ca17ec 100644
--- a/.github/workflows/tmpl_build_image_cache.yml
+++ b/.github/workflows/tmpl_build_image_cache.yml
@@ -43,7 +43,7 @@ jobs:
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to DockerHub
- uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 #
v4.4.0
+ uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c #
v4.5.0
with:
registry: ghcr.io
username: ${{ github.actor }}
diff --git a/.github/workflows/tmpl_cloud_aws.yml
b/.github/workflows/tmpl_cloud_aws.yml
index 18edf2bab3a..4937bdb69c7 100644
--- a/.github/workflows/tmpl_cloud_aws.yml
+++ b/.github/workflows/tmpl_cloud_aws.yml
@@ -38,6 +38,17 @@ on:
type: string
description: OS tag for runner (e.g., Linux, ubuntu-24.04)
default: ubuntu_24.04
+ checkout_repository:
+ required: false
+ type: string
+ description: Repository to checkout for this run
+ checkout_ref:
+ required: false
+ type: string
+ description: Ref or SHA to checkout for this run
+ secrets:
+ LOCALSTACK_CI_KEY:
+ required: true
# Security: Minimal defaults for workflow.
permissions: {}
@@ -61,10 +72,11 @@ jobs:
outputs:
build_image_url: ${{ steps.img.outputs.build_image_url }}
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 #
v6.1.0
with:
- # Full fetch so build image URL can be computed for any branch
- fetch-depth: 0
+ repository: ${{ inputs.checkout_repository || github.repository }}
+ ref: ${{ inputs.checkout_ref || github.ref }}
+ persist-credentials: false
- uses: ./.github/actions/build_image_url
id: img
with:
@@ -87,6 +99,10 @@ jobs:
run: |
echo "Build image URL: ${{
needs.precondition.outputs.build_image_url }}"
- uses: actions/checkout@v6
+ with:
+ repository: ${{ inputs.checkout_repository || github.repository }}
+ ref: ${{ inputs.checkout_ref || github.ref }}
+ persist-credentials: false
- uses: ./.github/actions/build_image
id: build_img
with:
@@ -141,6 +157,10 @@ jobs:
-Dcheckstyle.skip -Dspotbugs.skip -Denforcer.skip -Drat.skip
steps:
- uses: actions/checkout@v6
+ with:
+ repository: ${{ inputs.checkout_repository || github.repository }}
+ ref: ${{ inputs.checkout_ref || github.ref }}
+ persist-credentials: false
# Performance: Caching TODO: We need to create a centralized maven build
cache that is
# built on trunk. This will always miss on a new PR: Caches can't be
# shared between PR branches. PR branches *can* access caches from their
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
