LuciferYang commented on code in PR #13535: URL: https://github.com/apache/gluten/pull/13535#discussion_r4492807686
########## .github/workflows/update_fork_build_status.yml: ########## @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A pull request from a fork runs velox_backend_x86.yml in the fork, on push. This +# workflow reports that run on the pull request as the check named below: it creates +# the check when it first finds the run, and updates it on later passes. It only reads +# run metadata, so it never runs pull request code and needs no pull_request_target. +# Ported from apache/spark's update_build_status.yml (SPARK-60578). +name: Update fork build status + +on: + schedule: + - cron: '*/15 * * * *' + workflow_dispatch: + +permissions: {} + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + update: + if: github.repository == 'apache/gluten' + runs-on: ubuntu-slim + permissions: + actions: read + checks: write + steps: + - uses: actions/github-script@v9 + with: + script: | + const CHECK = 'Velox Backend (x86) in fork' + const WORKFLOW = 'velox_backend_x86.yml' + const repo = { owner: context.repo.owner, repo: context.repo.repo } + + // 5xx, network errors and rate limiting are retried on a later pass. + const isTransient = (e) => { + if (!e.status) return true + if (e.status >= 500 || e.status == 429) return true + if (e.status == 403) { + const h = (e.response && e.response.headers) || {} + return !!h['retry-after'] || h['x-ratelimit-remaining'] === '0' + || /rate limit/i.test(e.message || '') + } + return false + } + + const listChecks = (sha) => github.paginate( + 'GET /repos/{owner}/{repo}/commits/{ref}/check-runs', + { ...repo, ref: sha, check_name: CHECK, per_page: 100 }) + + // The fork run a check points at, or null for a check this workflow can't sync. + const runParams = (cr) => { + if (cr.conclusion == 'action_required') return null + try { + const p = JSON.parse(cr.output.text) + return p && p.owner && p.repo && p.run_id ? p : null + } catch (e) { + return null + } + } + + // PATCH accepts fewer values than a workflow run reports; anything else + // means the run produced no result. + const CONCLUSIONS = ['success', 'failure', 'neutral', 'cancelled', + 'timed_out', 'action_required', 'skipped'] + const syncCheck = (cr, run) => { + const body = { ...repo, check_run_id: cr.id, details_url: run.html_url } + if (run.status == 'completed') { + body.status = 'completed' + body.conclusion = CONCLUSIONS.includes(run.conclusion) ? run.conclusion : 'failure' + } else { + body.status = run.status == 'in_progress' ? 'in_progress' : 'queued' + } + if (cr.status == body.status && cr.conclusion == (body.conclusion || null)) return + console.log(` check ${cr.id}: ${body.status} ${body.conclusion || ''}`) + return github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', body) + } + + // The fork's run for the pull request's head commit, re-polled because a run + // can take a few seconds to register after a push. undefined: no run, including + // a fork that has no such workflow (404). + const findForkRun = async (pr, attempts) => { + for (let i = 0; i < attempts; i++) { + let runs + try { + const { data } = await github.request( + 'GET /repos/{owner}/{repo}/actions/workflows/{id}/runs', { + owner: pr.head.repo.owner.login, repo: pr.head.repo.name, + id: WORKFLOW, branch: pr.head.ref, head_sha: pr.head.sha + }) + runs = data.workflow_runs + } catch (e) { + if (e.status == 404) return undefined + throw e + } + const run = runs.find(r => r.event == 'push' && r.head_sha == pr.head.sha) + if (run) return run + if (i < attempts - 1) await new Promise(r => setTimeout(r, 3000)) + } + return undefined + } + + const createCheck = (pr, run) => github.rest.checks.create(run ? { + ...repo, name: CHECK, head_sha: pr.head.sha, status: 'queued', + details_url: run.html_url, + output: { + title: 'Fork run', + summary: `[See the run in ${pr.head.repo.full_name}](${run.html_url}). ` + + 'Rerun failed jobs there if the failure is unrelated to this pull request.', + text: JSON.stringify({ + owner: pr.head.repo.owner.login, repo: pr.head.repo.name, run_id: run.id }) + } + } : { + ...repo, name: CHECK, head_sha: pr.head.sha, + status: 'completed', conclusion: 'action_required', + output: { + title: 'No run found in your fork', + summary: [ + `${WORKFLOW} runs in your fork on each push to a branch other than main. No run was found for ${pr.head.sha}.`, + '', + '1. Enable GitHub Actions in the Actions tab of your fork, then push again. See [Managing GitHub Actions settings for a repository](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository).', + '2. If your branch predates fork runs, rebase it on the latest apache/gluten main and push again.', + '3. If this pull request comes from your fork\'s main branch, push it to another branch and open the pull request from there.' + ].join('\n') + } + }) + + const failed = [] + for await (const { data: prs } of github.paginate.iterator( + github.rest.pulls.list, { ...repo, state: 'open', base: 'main', per_page: 100 })) { + for (const pr of prs) { + // Branches of this repository (Dependabot) run upstream; skip deleted forks. + if (!pr.head.repo || pr.head.repo.full_name == `${repo.owner}/${repo.repo}`) continue + try { + const checks = await listChecks(pr.head.sha) + let synced = false + for (const cr of checks) { + const p = runParams(cr) + if (!p) continue + try { + const { data: run } = await github.request( + 'GET /repos/{owner}/{repo}/actions/runs/{run_id}', p) + await syncCheck(cr, run) + synced = true + } catch (e) { + // A 404 means the run was deleted or Actions disabled: fall through and + // create a fresh check. A transient error waits for the next pass. + if (isTransient(e)) synced = true + else if (e.status != 404) throw e + } + } + if (synced) continue + + // Once the action_required check exists, look once per pass, not three times. + const hasNoRunCheck = checks.some(cr => cr.conclusion == 'action_required') + const run = await findForkRun(pr, hasNoRunCheck ? 1 : 3) + if (!run && hasNoRunCheck) continue + console.log(`#${pr.number}: ${run ? 'tracking ' + run.html_url : 'no fork run'}`) + await createCheck(pr, run) + } catch (e) { + if (isTransient(e)) continue + console.error(`#${pr.number}: ${e.message}`) + failed.push(pr.number) + } + } + } + // Keep a recurring error visible instead of hiding it in a green run. + if (failed.length) core.setFailed('Failed to sync #' + failed.join(', #')) Review Comment: Done in 75fd0621d. The script is now `.github/workflows/util/update-fork-build-status.js`, and the workflow loads it with `require` the way `pr_bot.yml` loads `pr_issue_linker.js`. ########## .github/actions/squash-merge-upstream/action.yml: ########## @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Squash-merge onto apache/gluten main +description: >- + On a fork push, squash-merge the pushed commit onto the current apache/gluten + main, which is what a pull request merge ref would hold, and leave HEAD at the + result. If anything in `paths` changed, upload the merged tree for the + workflow's other jobs to swap in with .github/actions/checkout-merged. Use it + after actions/checkout with fetch-depth 0. +inputs: + paths: + description: >- + Extended regex for the files the workflow covers. A `paths:` filter on push + diffs against the branch's previous push, not against main, so fork runs + apply the list here. + required: true + paths-ignore: + description: Extended regex for files to leave out before matching `paths`. + default: '^$' +outputs: + base_sha: + description: The apache/gluten main commit the branch was merged onto. + value: ${{ steps.merge.outputs.base_sha }} + relevant: + description: "'false' if nothing in `paths` changed." + value: ${{ steps.merge.outputs.relevant }} +runs: + using: composite + steps: + - name: Squash-merge the pushed commit + id: merge + shell: bash + env: + PATHS: ${{ inputs.paths }} + PATHS_IGNORE: ${{ inputs.paths-ignore }} + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git fetch --no-tags https://github.com/apache/gluten.git main + base_sha=$(git rev-parse FETCH_HEAD) + git checkout --quiet --detach "$base_sha" + if ! git merge --squash --no-commit "$GITHUB_SHA"; then + echo "::error::$GITHUB_REF_NAME does not merge cleanly into apache/gluten main ($base_sha). Rebase it and push again." + exit 1 + fi + git commit --quiet --allow-empty -m "Squash-merge $GITHUB_SHA onto $base_sha" + echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" + if git diff --name-only "$base_sha" HEAD | grep -vE "$PATHS_IGNORE" | grep -qE "$PATHS"; then Review Comment: Fixed in 75fd0621d, the check now uses `grep -E ... > /dev/null`. With `grep -q`, a branch changing 6,001 files was skipped in 3 of 3 tries; with the fix it runs. ########## .github/actions/squash-merge-upstream/action.yml: ########## @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Squash-merge onto apache/gluten main +description: >- + On a fork push, squash-merge the pushed commit onto the current apache/gluten + main, which is what a pull request merge ref would hold, and leave HEAD at the + result. If anything in `paths` changed, upload the merged tree for the + workflow's other jobs to swap in with .github/actions/checkout-merged. Use it + after actions/checkout with fetch-depth 0. +inputs: + paths: + description: >- + Extended regex for the files the workflow covers. A `paths:` filter on push + diffs against the branch's previous push, not against main, so fork runs + apply the list here. + required: true + paths-ignore: + description: Extended regex for files to leave out before matching `paths`. + default: '^$' +outputs: + base_sha: + description: The apache/gluten main commit the branch was merged onto. + value: ${{ steps.merge.outputs.base_sha }} + relevant: + description: "'false' if nothing in `paths` changed." + value: ${{ steps.merge.outputs.relevant }} +runs: + using: composite + steps: + - name: Squash-merge the pushed commit + id: merge + shell: bash + env: + PATHS: ${{ inputs.paths }} + PATHS_IGNORE: ${{ inputs.paths-ignore }} + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git fetch --no-tags https://github.com/apache/gluten.git main + base_sha=$(git rev-parse FETCH_HEAD) + git checkout --quiet --detach "$base_sha" + if ! git merge --squash --no-commit "$GITHUB_SHA"; then + echo "::error::$GITHUB_REF_NAME does not merge cleanly into apache/gluten main ($base_sha). Rebase it and push again." + exit 1 + fi + git commit --quiet --allow-empty -m "Squash-merge $GITHUB_SHA onto $base_sha" + echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" + if git diff --name-only "$base_sha" HEAD | grep -vE "$PATHS_IGNORE" | grep -qE "$PATHS"; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "Nothing this workflow covers changed -- skipping." + echo "relevant=false" >> "$GITHUB_OUTPUT" + fi + - name: Pack the merged tree + if: steps.merge.outputs.relevant == 'true' + shell: bash + run: git archive --format=tar.gz -o "$RUNNER_TEMP/merged-tree.tar.gz" HEAD + - uses: actions/upload-artifact@v7 + if: steps.merge.outputs.relevant == 'true' + with: + name: merged-tree + path: ${{ runner.temp }}/merged-tree.tar.gz + compression-level: 0 + if-no-files-found: error + retention-days: 1 Review Comment: Fixed in 75fd0621d, `merged-tree` now uses the repository's default retention. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
