bitflicker64 commented on code in PR #3326: URL: https://github.com/apache/hugegraph/pull/3326#discussion_r4256493323
########## .github/workflows/submit-dependencies.yml: ########## @@ -0,0 +1,315 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Dependency baseline + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dependency-baseline-${{ github.repository }}-master + cancel-in-progress: false + +jobs: + validate: + if: >- + (github.repository == 'apache/hugegraph' || github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Checkout event commit + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Test CI policy and dependency inventory + run: | + python3 -m unittest discover -s .github/scripts -p 'test_ci_policy.py' + python3 -m unittest discover -s .github/scripts -p 'test_dependency_snapshot.py' + python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' + + generate: + needs: validate + if: >- + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + env: + PYTHONDONTWRITEBYTECODE: '1' + INPUT_TOKEN: '' + GITHUB_TOKEN: '' + GH_TOKEN: '' + outputs: + artifact_id: ${{ steps.snapshot.outputs.artifact-id }} + steps: + - name: Checkout event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Set up project JDK + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + java-version-file: .github/workflows/.java-version + distribution: 'temurin' + - name: Verify checkout and current master + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Install the complete Maven reactor + run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end + - name: Generate resolved reactor snapshot + run: | + mvn -B -ntp com.github.ferstl:depgraph-maven-plugin:4.0.3:graph \ + -DgraphFormat=json -DoutputFileName=maven-dependency-submission-action-depgraph.json \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-snapshot.py "$RUNNER_TEMP/snapshot.json" + - name: Refuse a stale generated snapshot + env: + EXPECTED_SHA: ${{ github.sha }} + run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Upload snapshot data only + id: snapshot + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dependency-snapshot-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/snapshot.json + if-no-files-found: error + retention-days: 1 + + submit: + needs: [validate, generate] + if: >- + needs.validate.result == 'success' && needs.generate.result == 'success' && + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + actions: read + steps: + - name: Validate and submit snapshot data + env: + SNAPSHOT_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} + run: | + python3 -I - <<'PY' + import datetime + import hashlib + import io + import json + import os + import re + import stat + import urllib.error + import urllib.parse + import urllib.request + import zipfile + + LIMIT = 25 * 1024 * 1024 + + def require(condition, message): + if not condition: + raise ValueError(message) + + def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'Duplicate JSON key') + result[key] = value + return result + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + def github_request(repository, path, data=None): + return urllib.request.Request('https://api.github.com/repos/' + repository + '/' + path, + data=None if data is None else json.dumps(data).encode(), + headers={'Authorization': 'Bearer ' + os.environ['SNAPSHOT_TOKEN'], + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28'}) + + def bounded_body(response): + raw = response.read(LIMIT + 1) + require(len(raw) <= LIMIT, 'Response too large') + return raw + + def api(repository, path, data=None): + request = github_request(repository, path, data) + with urllib.request.build_opener(NoRedirect()).open(request, timeout=60) as response: + require(response.status == (200 if data is None else 201), 'Unexpected API status') + raw = bounded_body(response) + return json.loads(raw, object_pairs_hook=unique_object) + + def read_snapshot(raw): + require(0 < len(raw) <= LIMIT, 'Invalid ZIP size') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + members = archive.infolist() + require(len(members) == 1, 'ZIP must contain one snapshot.json') + member = members[0] + require(member.orig_filename == 'snapshot.json' and not member.is_dir() and + not (member.external_attr & 0x10) and + stat.S_IFMT(member.external_attr >> 16) in [0, stat.S_IFREG] and + not (member.flag_bits & 1) and + member.compress_type in [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED] and + 0 < member.file_size <= LIMIT, 'Invalid ZIP member') + with archive.open(member) as stream: + payload = bounded_body(stream) + require(len(payload) == member.file_size, 'Incomplete ZIP member') + return json.loads(payload, object_pairs_hook=unique_object) + + def download_snapshot(repository, artifact_id, artifact): + opener = urllib.request.build_opener(NoRedirect()) + try: + response = opener.open(github_request(repository, f'actions/artifacts/{artifact_id}/zip'), timeout=60) + except urllib.error.HTTPError as error: + require(error.code == 302, 'Expected artifact redirect') + location = error.headers.get('Location', '') + error.close() + else: + response.close() + raise ValueError('Expected artifact redirect') + target = urllib.parse.urlsplit(location) + require(target.scheme == 'https' and target.hostname and + target.hostname.endswith(('.blob.core.windows.net', '.githubusercontent.com')) and + target.username is None and target.password is None and + target.port in [None, 443] and not target.fragment, 'Invalid artifact redirect') + # A separate request carries no GitHub Authorization to blob storage. + with opener.open(urllib.request.Request(location), timeout=60) as response: + require(response.status == 200, 'Artifact download failed') + raw = bounded_body(response) + require(len(raw) == artifact['size_in_bytes'] and + 'sha256:' + hashlib.sha256(raw).hexdigest() == artifact['digest'], 'Artifact digest/size mismatch') + return read_snapshot(raw) + + def timestamp(value): + parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + require(parsed.tzinfo is not None, 'Timestamp lacks timezone') + return parsed + + def main(): + repository, sha = os.environ['EXPECTED_REPOSITORY'], os.environ['EXPECTED_SHA'] + run_id, attempt, artifact_id = (os.environ[key] for key in ['RUN_ID', 'RUN_ATTEMPT', 'ARTIFACT_ID']) + require(repository in ['apache/hugegraph', 'hugegraph/hugegraph'], 'Repository refused') + require(re.fullmatch('[0-9a-f]{40}', sha), 'Invalid commit') + require(all(re.fullmatch('[1-9][0-9]*', x) for x in [run_id, attempt, artifact_id]), 'Invalid ID') + run = api(repository, f'actions/runs/{run_id}/attempts/{attempt}') + require(run['id'] == int(run_id) and run['run_attempt'] == int(attempt) and + run['head_sha'] == sha and run['head_branch'] == 'master' and + run['event'] in ['push', 'workflow_dispatch'] and + run['repository']['full_name'] == repository and + run['path'] in ['.github/workflows/submit-dependencies.yml', + '.github/workflows/submit-dependencies.yml@master'], 'Run identity') + jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') + producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] + require(jobs['total_count'] <= 100 and len(producers) == 1 and + producers[0]['conclusion'] == 'success' and producers[0]['status'] == 'completed' and + producers[0]['run_id'] == int(run_id) and producers[0]['head_sha'] == sha, + 'Generation did not succeed for this commit') + artifact = api(repository, f'actions/artifacts/{artifact_id}') + require(artifact['id'] == int(artifact_id) and artifact['expired'] is False and + artifact['name'] == f'dependency-snapshot-{run_id}-{attempt}' and Review Comment: Minor: a failed `submit` cannot be recovered with "Re-run failed jobs", and `docs/ci.md` does not say so. Evidence: `generate` names the artifact `dependency-snapshot-<run_id>-<run_attempt>` (line 105) and stamps `metadata.run_attempt` with its own attempt. When only `submit` is re-run, `generate` is not executed again, so the artifact and snapshot still carry attempt 1 while `RUN_ATTEMPT` in the writer is 2. The name check here then fails with `Artifact identity`, and the `Producer identity` check (line 268) and the `run_started_at <= scanned` freshness check would fail for the same reason. So a transient failure of the snapshot POST or of an API read leaves the maintainer with a re-run that fails every time until all jobs are re-run. Failing closed is reasonable; the gap is that nothing tells the operator. Requested change: add a sentence to the Dependency baseline section of `docs/ci.md` saying a failed submission needs "Re-run all jobs" (or a fresh `workflow_dispatch` on master), because the writer only accepts a snapshot produced in the same run attempt. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
