This is an automated email from the ASF dual-hosted git repository.
dpgaspar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new e4bdb28ba2 fix(ci): change ephemeral env to use github labels instead
of comments (#31340)
e4bdb28ba2 is described below
commit e4bdb28ba22367bf08a79544087a2bb98629c6ce
Author: Daniel Vaz Gaspar <[email protected]>
AuthorDate: Wed Jan 29 13:57:01 2025 +0000
fix(ci): change ephemeral env to use github labels instead of comments
(#31340)
---
.github/workflows/ephemeral-env.yml | 425 +++++++++++++++++-------------------
1 file changed, 202 insertions(+), 223 deletions(-)
diff --git a/.github/workflows/ephemeral-env.yml
b/.github/workflows/ephemeral-env.yml
index 91a3b54b73..4c4747f5e2 100644
--- a/.github/workflows/ephemeral-env.yml
+++ b/.github/workflows/ephemeral-env.yml
@@ -1,145 +1,124 @@
name: Ephemeral env workflow
-# Example manual trigger: gh workflow run ephemeral-env.yml --ref
fix_ephemerals --field comment_body="/testenv up" --field issue_number=666
+# Example manual trigger:
+# gh workflow run ephemeral-env.yml --ref fix_ephemerals --field
label_name="testenv-up" --field issue_number=666
on:
- issue_comment:
- types: [created]
+ pull_request_target:
+ types:
+ - labeled
workflow_dispatch:
inputs:
- comment_body:
- description: 'Comment body to simulate /testenv command'
+ label_name:
+ description: 'Label name to simulate label-based /testenv trigger'
required: true
- default: '/testenv up'
+ default: 'testenv-up'
issue_number:
description: 'Issue or PR number'
required: true
jobs:
- ephemeral-env-comment:
+ ephemeral-env-label:
concurrency:
- group: ${{ github.workflow }}-${{ github.event.inputs.issue_number ||
github.event.issue.number || github.run_id }}-comment
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.run_id }}-label
cancel-in-progress: true
- name: Evaluate ephemeral env comment trigger (/testenv)
+ name: Evaluate ephemeral env label trigger
runs-on: ubuntu-24.04
permissions:
pull-requests: write
outputs:
- slash-command: ${{ steps.eval-body.outputs.result }}
+ slash-command: ${{ steps.eval-label.outputs.result }}
feature-flags: ${{ steps.eval-feature-flags.outputs.result }}
+ sha: ${{ steps.get-sha.outputs.sha }}
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
steps:
- - name: Debug
- run: |
- echo "Comment on PR #${{ github.event.issue.number }} by ${{
github.event.issue.user.login }}, ${{ github.event.comment.author_association
}}"
-
- - name: Eval comment body for /testenv slash command
- uses: actions/github-script@v7
- env:
- COMMENT_BODY: ${{ github.event.inputs.comment_body ||
github.event.comment.body }}
- id: eval-body
- with:
- result-encoding: string
- script: |
- const pattern = /^\/testenv (up|down)/;
- const result = pattern.exec(process.env.COMMENT_BODY || '');
- return result === null ? 'noop' : result[1];
-
- - name: Looking for feature flags
- uses: actions/github-script@v7
- env:
- COMMENT_BODY: ${{ github.event.inputs.comment_body ||
github.event.comment.body }}
- id: eval-feature-flags
- with:
- script: |
- const pattern = /FEATURE_(\w+)=(\w+)/g;
- let results = [];
- [...process.env.COMMENT_BODY.matchAll(pattern)].forEach(match => {
- const config = {
- name: `SUPERSET_FEATURE_${match[1]}`,
- value: match[2],
- };
- results.push(config);
- });
- return results;
-
- - name: Limit to committers
- if: >
- steps.eval-body.outputs.result != 'noop' &&
- github.event_name == 'issue_comment' &&
- github.event.comment.author_association != 'MEMBER' &&
- github.event.comment.author_association != 'OWNER'
- uses: actions/github-script@v7
- with:
- github-token: ${{ github.token }}
- script: |
- const errMsg = '@${{ github.event.comment.user.login }} Ephemeral
environment creation is currently limited to committers.';
- github.rest.issues.createComment({
- issue_number: ${{ github.event.issue.number }},
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: errMsg
- });
- core.setFailed(errMsg);
-
- - name: Reply with confirmation comment
- uses: actions/github-script@v7
- with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- script: |
- const issueNumber = ${{ github.event.inputs.issue_number ||
github.event.issue.number }};
- const user = '${{ github.event.comment.user.login || github.actor
}}';
- const action = '${{ steps.eval-body.outputs.result }}';
- const runId = context.runId;
- const workflowUrl =
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
- const body = `@${user} Processing your ephemeral environment request
[here](${workflowUrl}).`;
- if (action !== 'noop') {
+ - name: Check for the "testenv-up" label
+ id: eval-label
+ run: |
+ if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+ LABEL_NAME="${{ github.event.inputs.label_name }}"
+ else
+ LABEL_NAME="${{ github.event.label.name }}"
+ fi
+
+ echo "Evaluating label: $LABEL_NAME"
+
+ if [[ "$LABEL_NAME" == "testenv-up" ]]; then
+ echo "result=up" >> $GITHUB_OUTPUT
+ else
+ echo "result=noop" >> $GITHUB_OUTPUT
+ exit 1
+ fi
+
+ - name: Get event SHA
+ id: get-sha
+ run: |
+ echo "sha=${{ github.event.pull_request.head.sha }}" >>
$GITHUB_OUTPUT
+
+ - name: Looking for feature flags in PR description
+ uses: actions/github-script@v7
+ id: eval-feature-flags
+ with:
+ script: |
+ const description = context.payload.pull_request
+ ? context.payload.pull_request.body || ''
+ : context.payload.inputs.pr_description || '';
+
+ const pattern = /FEATURE_(\w+)=(\w+)/g;
+ let results = [];
+ [...description.matchAll(pattern)].forEach(match => {
+ const config = {
+ name: `SUPERSET_FEATURE_${match[1]}`,
+ value: match[2],
+ };
+ results.push(config);
+ });
+
+ return results;
+
+ - name: Reply with confirmation comment
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const action = '${{ steps.eval-label.outputs.result }}';
+ const user = context.actor;
+ const runId = context.runId;
+ const workflowUrl =
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
+
+ const issueNumber = context.payload.pull_request
+ ? context.payload.pull_request.number
+ : context.payload.inputs.issue_number;
+
+ if (!issueNumber) {
+ throw new Error("Issue number is not available.");
+ }
+
+ const body = `@${user} Processing your ephemeral environment
request [here](${workflowUrl}). Action: **${action}**.`;
+
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
body,
});
- }
- else {
- core.setFailed('No ephemeral environment action detected.');
- }
ephemeral-docker-build:
concurrency:
- group: ${{ github.workflow }}-${{ github.event.inputs.issue_number ||
github.event.issue.number || github.run_id }}-build
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.run_id }}-build
cancel-in-progress: true
- needs: ephemeral-env-comment
- if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
+ needs: ephemeral-env-label
+ if: needs.ephemeral-env-label.outputs.slash-command == 'up'
name: ephemeral-docker-build
runs-on: ubuntu-24.04
steps:
- - name: Get Info from comment
- uses: actions/github-script@v7
- id: get-pr-info
- with:
- script: |
- const request = {
- owner: context.repo.owner,
- repo: context.repo.repo,
- pull_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
- };
- core.info(`Getting PR #${request.pull_number} from
${request.owner}/${request.repo}`);
- const pr = await github.rest.pulls.get(request);
- return pr.data;
-
- - name: Debug
- id: get-sha
- run: |
- echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha
}}" >> $GITHUB_OUTPUT
-
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} :
${{steps.get-sha.outputs.sha}} )"
+ - name: "Checkout ${{ github.ref }} ( ${{
needs.ephemeral-env-label.outputs.sha }} : ${{steps.get-sha.outputs.sha}} )"
uses: actions/checkout@v4
with:
- ref: ${{ steps.get-sha.outputs.sha }}
+ ref: ${{ needs.ephemeral-env-label.outputs.sha }}
persist-credentials: false
- name: Setup Docker Environment
@@ -181,14 +160,14 @@ jobs:
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: superset-ci
- IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
+ IMAGE_TAG: apache/superset:${{ needs.ephemeral-env-label.outputs.sha
}}-ci
run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{
github.event.inputs.issue_number || github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up:
- needs: [ephemeral-env-comment, ephemeral-docker-build]
- if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
+ needs: [ephemeral-env-label, ephemeral-docker-build]
+ if: needs.ephemeral-env-label.outputs.slash-command == 'up'
name: Spin up an ephemeral environment
runs-on: ubuntu-24.04
permissions:
@@ -196,120 +175,120 @@ jobs:
pull-requests: write
steps:
- - uses: actions/checkout@v4
- with:
- persist-credentials: false
-
- - name: Configure AWS credentials
- uses: aws-actions/configure-aws-credentials@v4
- with:
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- aws-region: us-west-2
-
- - name: Login to Amazon ECR
- id: login-ecr
- uses: aws-actions/amazon-ecr-login@v2
-
- - name: Check target image exists in ECR
- id: check-image
- continue-on-error: true
- run: |
- aws ecr describe-images \
- --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep
-Eo "^[0-9]+") \
- --repository-name superset-ci \
- --image-ids imageTag=pr-${{ github.event.inputs.issue_number ||
github.event.issue.number }}-ci
-
- - name: Fail on missing container image
- if: steps.check-image.outcome == 'failure'
- uses: actions/github-script@v7
- with:
- github-token: ${{ github.token }}
- script: |
- const errMsg = '@${{ github.event.comment.user.login }} Container
image not yet published for this PR. Please try again when build is complete.';
- github.rest.issues.createComment({
- issue_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: errMsg
- });
- core.setFailed(errMsg);
-
- - name: Fill in the new image ID in the Amazon ECS task definition
- id: task-def
- uses: aws-actions/amazon-ecs-render-task-definition@v1
- with:
- task-definition: .github/workflows/ecs-task-definition.json
- container-name: superset-ci
- image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{
github.event.inputs.issue_number || github.event.issue.number }}-ci
-
- - name: Update env vars in the Amazon ECS task definition
- run: |
- cat <<< "$(jq '.containerDefinitions[0].environment += ${{
needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{
steps.task-def.outputs.task-definition }})" > ${{
steps.task-def.outputs.task-definition }}
-
- - name: Describe ECS service
- id: describe-services
- run: |
- echo "active=$(aws ecs describe-services --cluster superset-ci
--services pr-${{ github.event.inputs.issue_number || github.event.issue.number
}}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >>
$GITHUB_OUTPUT
- - name: Create ECS service
- id: create-service
- if: steps.describe-services.outputs.active != 'true'
- env:
- ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
- ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
- run: |
- aws ecs create-service \
- --cluster superset-ci \
- --service-name pr-${{ github.event.inputs.issue_number ||
github.event.issue.number }}-service \
- --task-definition superset-ci \
- --launch-type FARGATE \
- --desired-count 1 \
- --platform-version LATEST \
- --network-configuration
"awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}"
\
- --tags key=pr,value=${{ github.event.inputs.issue_number ||
github.event.issue.number }} key=github_user,value=${{ github.actor }}
- - name: Deploy Amazon ECS task definition
- id: deploy-task
- uses: aws-actions/amazon-ecs-deploy-task-definition@v2
- with:
- task-definition: ${{ steps.task-def.outputs.task-definition }}
- service: pr-${{ github.event.inputs.issue_number ||
github.event.issue.number }}-service
- cluster: superset-ci
- wait-for-service-stability: true
- wait-for-minutes: 10
-
- - name: List tasks
- id: list-tasks
- run: |
- echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name
pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service
| jq '.taskArns | first')" >> $GITHUB_OUTPUT
- - name: Get network interface
- id: get-eni
- run: |
- echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{
steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] |
.details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >>
$GITHUB_OUTPUT
- - name: Get public IP
- id: get-ip
- run: |
- echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids
${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first |
.Association.PublicIp')" >> $GITHUB_OUTPUT
- - name: Comment (success)
- if: ${{ success() }}
- uses: actions/github-script@v7
- with:
- github-token: ${{github.token}}
- script: |
- github.rest.issues.createComment({
- issue_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: '@${{ github.event.inputs.user_login ||
github.event.comment.user.login }} Ephemeral environment spinning up at
http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`.
Please allow several minutes for bootstrapping and startup.'
- })
- - name: Comment (failure)
- if: ${{ failure() }}
- uses: actions/github-script@v7
- with:
- github-token: ${{github.token}}
- script: |
- github.rest.issues.createComment({
- issue_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: '@${{ github.event.inputs.user_login ||
github.event.comment.user.login }} Ephemeral environment creation failed.
Please check the Actions logs for details.'
- })
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: us-west-2
+
+ - name: Login to Amazon ECR
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@v2
+
+ - name: Check target image exists in ECR
+ id: check-image
+ continue-on-error: true
+ run: |
+ aws ecr describe-images \
+ --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" |
grep -Eo "^[0-9]+") \
+ --repository-name superset-ci \
+ --image-ids imageTag=pr-${{ github.event.inputs.issue_number ||
github.event.issue.number }}-ci
+
+ - name: Fail on missing container image
+ if: steps.check-image.outcome == 'failure'
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{ github.token }}
+ script: |
+ const errMsg = '@${{ github.event.comment.user.login }} Container
image not yet published for this PR. Please try again when build is complete.';
+ github.rest.issues.createComment({
+ issue_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: errMsg
+ });
+ core.setFailed(errMsg);
+
+ - name: Fill in the new image ID in the Amazon ECS task definition
+ id: task-def
+ uses: aws-actions/amazon-ecs-render-task-definition@v1
+ with:
+ task-definition: .github/workflows/ecs-task-definition.json
+ container-name: superset-ci
+ image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{
github.event.inputs.issue_number || github.event.issue.number }}-ci
+
+ - name: Update env vars in the Amazon ECS task definition
+ run: |
+ cat <<< "$(jq '.containerDefinitions[0].environment += ${{
needs.ephemeral-env-label.outputs.feature-flags }}' < ${{
steps.task-def.outputs.task-definition }})" > ${{
steps.task-def.outputs.task-definition }}
+
+ - name: Describe ECS service
+ id: describe-services
+ run: |
+ echo "active=$(aws ecs describe-services --cluster superset-ci
--services pr-${{ github.event.inputs.issue_number || github.event.issue.number
}}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >>
$GITHUB_OUTPUT
+ - name: Create ECS service
+ id: create-service
+ if: steps.describe-services.outputs.active != 'true'
+ env:
+ ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
+ ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
+ run: |
+ aws ecs create-service \
+ --cluster superset-ci \
+ --service-name pr-${{ github.event.inputs.issue_number ||
github.event.issue.number }}-service \
+ --task-definition superset-ci \
+ --launch-type FARGATE \
+ --desired-count 1 \
+ --platform-version LATEST \
+ --network-configuration
"awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}"
\
+ --tags key=pr,value=${{ github.event.inputs.issue_number ||
github.event.issue.number }} key=github_user,value=${{ github.actor }}
+ - name: Deploy Amazon ECS task definition
+ id: deploy-task
+ uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+ with:
+ task-definition: ${{ steps.task-def.outputs.task-definition }}
+ service: pr-${{ github.event.inputs.issue_number ||
github.event.issue.number }}-service
+ cluster: superset-ci
+ wait-for-service-stability: true
+ wait-for-minutes: 10
+
+ - name: List tasks
+ id: list-tasks
+ run: |
+ echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name
pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service
| jq '.taskArns | first')" >> $GITHUB_OUTPUT
+ - name: Get network interface
+ id: get-eni
+ run: |
+ echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{
steps.list-tasks.outputs.task }} | jq '.tasks[0].attachments[0].details |
map(select(.name==\"networkInterfaceId\"))[0].value')" >> $GITHUB_OUTPUT
+ - name: Get public IP
+ id: get-ip
+ run: |
+ echo "ip=$(aws ec2 describe-network-interfaces
--network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r
'.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
+ - name: Comment (success)
+ if: ${{ success() }}
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{github.token}}
+ script: |
+ github.rest.issues.createComment({
+ issue_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: '@${{ github.actor }} Ephemeral environment spinning up at
http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`.
Please allow several minutes for bootstrapping and startup.'
+ })
+ - name: Comment (failure)
+ if: ${{ failure() }}
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{github.token}}
+ script: |
+ github.rest.issues.createComment({
+ issue_number: ${{ github.event.inputs.issue_number ||
github.event.issue.number }},
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: '@${{ github.event.inputs.user_login ||
github.event.comment.user.login }} Ephemeral environment creation failed.
Please check the Actions logs for details.'
+ })
