This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-cleanup-trusted-main in repository https://gitbox.apache.org/repos/asf/tvm.git
commit bd5449872b507f93b215f8e6044489406b532c5a Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 02:16:59 2026 +0000 [REFACTOR][CI] Retire stale scripts and recover rejected helpers from main --- ci/README.md | 3 +- ci/jenkins/README.md | 45 +- ci/jenkins/generated/arm_jenkinsfile.groovy | 39 +- ci/jenkins/generated/cpu_jenkinsfile.groovy | 41 +- ci/jenkins/generated/docker_jenkinsfile.groovy | 39 +- ci/jenkins/generated/gpu_jenkinsfile.groovy | 39 +- ci/jenkins/generated/wasm_jenkinsfile.groovy | 39 +- ci/scripts/github/__init__.py | 18 - ci/scripts/github/github_tvmbot.py | 799 --------------------- ci/scripts/jenkins/cmd_utils.py | 107 --- ci/scripts/jenkins/determine_docker_images.py | 153 ---- ci/scripts/jenkins/git_change_docker.sh | 55 -- ci/scripts/jenkins/git_change_docs.sh | 39 - ci/scripts/jenkins/git_skip_ci.py | 61 -- ci/scripts/jenkins/git_skip_ci_globs.py | 76 -- ci/scripts/jenkins/git_utils.py | 212 ------ ci/scripts/jenkins/http_utils.py | 34 - ci/scripts/jenkins/retry.sh | 39 - ci/scripts/jenkins/s3.py | 193 ----- ci/scripts/jenkins/should_rebuild_docker.py | 152 ---- ci/scripts/jenkins/should_run_slow_tests.py | 85 --- ci/scripts/package/README.md | 31 - ci/scripts/package/build-environment.yaml | 42 -- .../package/manylinux_build_libtvm_runtime_cuda.sh | 64 -- .../package/windows_build_libtvm_runtime_cuda.bat | 99 --- tests/python/ci/jenkins_loader_test.groovy | 94 +++ tests/python/ci/test_ci.py | 57 +- tests/python/ci/test_jenkins_helpers.py | 8 +- tests/python/ci/test_jenkins_loader.py | 106 +++ tests/scripts/task_show_node_info.sh | 44 -- 30 files changed, 418 insertions(+), 2395 deletions(-) diff --git a/ci/README.md b/ci/README.md index 163d600dcd..292fdddd61 100644 --- a/ci/README.md +++ b/ci/README.md @@ -27,8 +27,7 @@ TVM project in a healthy state and preventing breakages. CI in TVM is broken int - Definitions of test suites, with each suite defined as a separate `task_` script in [`tests/scripts`](../tests/scripts). - GitHub automation [`ci/github`](github), wheel builds [`ci/wheel`](wheel), - and Jenkins host commands [`ci/jenkins/scripts`](jenkins/scripts). Legacy - [`ci/scripts`](scripts) paths remain for older trusted Jenkinsfiles used by PR jobs. + and Jenkins host commands [`ci/jenkins/scripts`](jenkins/scripts). - The linux test sequence (in [`Jenkinsfile`](../ci/jenkins/generated/)), which lints and builds TVM and runs test suites using Docker on Linux. - The Windows and Mac test sequences (in [`.github/actions`](../.github/actions)). diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 052260d4ef..495ff64cb8 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -54,32 +54,35 @@ and the standalone skip commands retain their command-line interfaces. ### Trust and rollout -Each entry point evaluates the shared helper returned by `readTrusted` at -Jenkins' selected SCM revision. Untrusted PRs that modify or delete the helper -fail immediately, before node allocation, checkout, or credentials. Missing -helpers, interrupted reads, and evaluation errors also propagate directly. -No alternative helper is fetched or evaluated. - -Trusted helper changes use Jenkins' selected revision directly. Cancellation -runs once before preparation. During preparation, the normal source merge pins -the PR target revision for subsequent stages, including release targets. -Host scripts and artifact metadata follow that pinned target revision for PR -authors outside `CONTRIBUTORS.md`. - -The legacy `ci/scripts/` hierarchy and `tests/scripts/task_show_node_info.sh` -remain alongside the new paths because fork PR jobs can select older trusted -Jenkinsfiles from their target branch. Those files keep the old docs-only and -slow-test gates and their helper imports; the native pipelines use the new -layout without those gates. Both layouts share the artifact registry and image -configuration. Keep the legacy commands until the trusted targets no longer -call them. Use a trusted branch build to exercise native helper changes; do not -bypass a refused read by evaluating PR-owned code or a different revision. +Each entry point first reads the shared helper with `readTrusted` at Jenkins' +selected SCM revision. Trusted named branches retain their own helper. For PRs, +Jenkins rejects an untrusted helper modification and may report a deleted helper +as missing. These specific read failures recover by fetching `refs/heads/main` +from the fixed `https://github.com/apache/tvm.git` URL into a fresh temporary bare +repository on a small CPU node. The fetched commit is resolved and its helper +blob is read directly, without checking out or evaluating PR-owned code. Recovery +always uses latest upstream main, including for release-target PRs. + +Other read failures, cancellation, fallback fetch/read failures and helper +evaluation errors propagate. Evaluation occurs outside the read-recovery block. +The fallback requires no credentials or controller configuration changes. + +Cancellation runs once before preparation. During preparation, the normal source +merge pins the PR target revision for subsequent stages, including release +targets. Host scripts and artifact metadata follow that pinned target revision +for PR authors outside `CONTRIBUTORS.md`. + +The obsolete `ci/scripts/` copies and old node-info path have been removed now +that main's Jenkinsfiles use the canonical layout. The live `task_build.py`, +source-mode test scripts and artifact interfaces remain available. Older target +branches must retain their matching legacy commands; they are not supplied by +this checkout. Before enabling the migration on production jobs, validate it with the existing Jenkins plugin versions and sandbox settings on a trusted branch: run all five entry points, suspend/resume across a node step, and exercise spot loss, a normal build-command failure, superseded builds, and a failed on-demand retry. Check -trusted helper changes, untrusted PRs without helper changes, and refusal of +trusted helper changes, untrusted PRs without helper changes, and recovery of untrusted modified/deleted helpers, including a release-target PR. Verify image overrides and disabled deployment gates before checking enabled docs/Docker deployment in the normal trusted deployment context. diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index 1b7c0d733d..e34ad8fdf2 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -15,8 +15,43 @@ // specific language governing permissions and limitations // under the License. -// Reject untrusted helper edits before allocating nodes or running pipeline steps. -ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) +// Keep trusted branches on their selected helper; recover rejected PR helpers +// only from the fixed upstream main ref. Never evaluate the checkout's copy. +def helperPath = 'ci/jenkins/generated/common.groovy' +def helperSource +try { + helperSource = readTrusted(helperPath) +} catch (hudson.AbortException failure) { + def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" + def missing = failure.message?.endsWith("/${helperPath} not found") + if (!env.CHANGE_ID || !(rejected || missing)) { + throw failure + } +} catch (java.io.FileNotFoundException failure) { + if (!env.CHANGE_ID || failure.message != helperPath) { + throw failure + } +} +if (helperSource == null) { + node('CPU-SMALL') { + helperSource = sh( + returnStdout: true, + label: 'Read trusted helper from upstream main', + script: ''' + set -eu + helper_repo=$(mktemp -d) + trap 'rm -rf "$helper_repo"' EXIT + git init --bare --quiet "$helper_repo" + git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ + https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main + revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') + git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" + ''' + ) + } +} +// Evaluation and subsequent pipeline failures must never trigger a fallback. +ci = evaluate(helperSource) ci.start() ci.build('ARM-GRAVITON3', 'BUILD: arm', 'tvm/build-arm', images.ci_arm, 'arm') { diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 02c2c234b4..119fd242a6 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -15,8 +15,43 @@ // specific language governing permissions and limitations // under the License. -// Reject untrusted helper edits before allocating nodes or running pipeline steps. -ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) +// Keep trusted branches on their selected helper; recover rejected PR helpers +// only from the fixed upstream main ref. Never evaluate the checkout's copy. +def helperPath = 'ci/jenkins/generated/common.groovy' +def helperSource +try { + helperSource = readTrusted(helperPath) +} catch (hudson.AbortException failure) { + def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" + def missing = failure.message?.endsWith("/${helperPath} not found") + if (!env.CHANGE_ID || !(rejected || missing)) { + throw failure + } +} catch (java.io.FileNotFoundException failure) { + if (!env.CHANGE_ID || failure.message != helperPath) { + throw failure + } +} +if (helperSource == null) { + node('CPU-SMALL') { + helperSource = sh( + returnStdout: true, + label: 'Read trusted helper from upstream main', + script: ''' + set -eu + helper_repo=$(mktemp -d) + trap 'rm -rf "$helper_repo"' EXIT + git init --bare --quiet "$helper_repo" + git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ + https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main + revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') + git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" + ''' + ) + } +} +// Evaluation and subsequent pipeline failures must never trigger a fallback. +ci = evaluate(helperSource) ci.start() ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') { @@ -31,7 +66,7 @@ ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') { stage('Test') { parallel('unittest: CPU': { - ci.test('CPU-SMALL', 'unittest: CPU', 'tvm/ut-python-cpu', images.ci_cpu, 'cpu') { + ci.test('CPU', 'unittest: CPU', 'tvm/ut-python-cpu', images.ci_cpu, 'cpu') { ci.download('cpu') ci.cpp_unittest(images.ci_cpu) ci.python_unittest(images.ci_cpu) diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index 0039be7712..b4b67852ba 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -15,8 +15,43 @@ // specific language governing permissions and limitations // under the License. -// Reject untrusted helper edits before allocating nodes or running pipeline steps. -ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) +// Keep trusted branches on their selected helper; recover rejected PR helpers +// only from the fixed upstream main ref. Never evaluate the checkout's copy. +def helperPath = 'ci/jenkins/generated/common.groovy' +def helperSource +try { + helperSource = readTrusted(helperPath) +} catch (hudson.AbortException failure) { + def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" + def missing = failure.message?.endsWith("/${helperPath} not found") + if (!env.CHANGE_ID || !(rejected || missing)) { + throw failure + } +} catch (java.io.FileNotFoundException failure) { + if (!env.CHANGE_ID || failure.message != helperPath) { + throw failure + } +} +if (helperSource == null) { + node('CPU-SMALL') { + helperSource = sh( + returnStdout: true, + label: 'Read trusted helper from upstream main', + script: ''' + set -eu + helper_repo=$(mktemp -d) + trap 'rm -rf "$helper_repo"' EXIT + git init --bare --quiet "$helper_repo" + git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ + https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main + revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') + git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" + ''' + ) + } +} +// Evaluation and subsequent pipeline failures must never trigger a fallback. +ci = evaluate(helperSource) ci.start() def ecr_push(full_name) { diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index ea49bfab06..47ccb0b5c0 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -15,8 +15,43 @@ // specific language governing permissions and limitations // under the License. -// Reject untrusted helper edits before allocating nodes or running pipeline steps. -ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) +// Keep trusted branches on their selected helper; recover rejected PR helpers +// only from the fixed upstream main ref. Never evaluate the checkout's copy. +def helperPath = 'ci/jenkins/generated/common.groovy' +def helperSource +try { + helperSource = readTrusted(helperPath) +} catch (hudson.AbortException failure) { + def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" + def missing = failure.message?.endsWith("/${helperPath} not found") + if (!env.CHANGE_ID || !(rejected || missing)) { + throw failure + } +} catch (java.io.FileNotFoundException failure) { + if (!env.CHANGE_ID || failure.message != helperPath) { + throw failure + } +} +if (helperSource == null) { + node('CPU-SMALL') { + helperSource = sh( + returnStdout: true, + label: 'Read trusted helper from upstream main', + script: ''' + set -eu + helper_repo=$(mktemp -d) + trap 'rm -rf "$helper_repo"' EXIT + git init --bare --quiet "$helper_repo" + git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ + https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main + revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') + git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" + ''' + ) + } +} +// Evaluation and subsequent pipeline failures must never trigger a fallback. +ci = evaluate(helperSource) ci.start() ci.build('CPU', 'BUILD: GPU', 'tvm/build-gpu', images.ci_gpu, 'gpu') { diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index ddf932529d..ac4b075721 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -15,8 +15,43 @@ // specific language governing permissions and limitations // under the License. -// Reject untrusted helper edits before allocating nodes or running pipeline steps. -ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) +// Keep trusted branches on their selected helper; recover rejected PR helpers +// only from the fixed upstream main ref. Never evaluate the checkout's copy. +def helperPath = 'ci/jenkins/generated/common.groovy' +def helperSource +try { + helperSource = readTrusted(helperPath) +} catch (hudson.AbortException failure) { + def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" + def missing = failure.message?.endsWith("/${helperPath} not found") + if (!env.CHANGE_ID || !(rejected || missing)) { + throw failure + } +} catch (java.io.FileNotFoundException failure) { + if (!env.CHANGE_ID || failure.message != helperPath) { + throw failure + } +} +if (helperSource == null) { + node('CPU-SMALL') { + helperSource = sh( + returnStdout: true, + label: 'Read trusted helper from upstream main', + script: ''' + set -eu + helper_repo=$(mktemp -d) + trap 'rm -rf "$helper_repo"' EXIT + git init --bare --quiet "$helper_repo" + git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ + https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main + revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') + git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" + ''' + ) + } +} +// Evaluation and subsequent pipeline failures must never trigger a fallback. +ci = evaluate(helperSource) ci.start() ci.build('CPU', 'BUILD: WASM', 'tvm/build-wasm', images.ci_wasm, 'wasm') { diff --git a/ci/scripts/github/__init__.py b/ci/scripts/github/__init__.py deleted file mode 100644 index fcdda82fc0..0000000000 --- a/ci/scripts/github/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# isort: skip_file -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Package to enable testing of GitHub scripts.""" diff --git a/ci/scripts/github/github_tvmbot.py b/ci/scripts/github/github_tvmbot.py deleted file mode 100755 index 6975bc9d35..0000000000 --- a/ci/scripts/github/github_tvmbot.py +++ /dev/null @@ -1,799 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E402, E501, F841, RUF012 - -import argparse -import json -import logging -import os -import re -import sys -import traceback -import warnings -from collections.abc import Callable -from pathlib import Path -from typing import Any - -# Hackery to enable importing of utils from ci/scripts/jenkins -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) - -from cmd_utils import init_log -from git_utils import GitHubRepo, git, parse_remote, post - -Review = dict[str, Any] -CIJob = dict[str, Any] -Comment = dict[str, Any] -CommentChecker = Callable[[Comment], bool] - -EXPECTED_JOBS = ["tvm-ci/pr-head"] -TVM_BOT_JENKINS_TOKEN = os.environ["TVM_BOT_JENKINS_TOKEN"] -GH_ACTIONS_TOKEN = os.environ["GH_ACTIONS_TOKEN"] -JENKINS_URL = "https://ci.tlcpack.ai/"; -THANKS_MESSAGE = r"(\s*)Thanks for contributing to TVM! Please refer to guideline https://tvm.apache.org/docs/contribute/ for useful information and tips. After the pull request is submitted, please request code reviews from \[Reviewers\]\(https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers\) by them in the pull request thread.(\s*)" - - -def to_json_str(obj: Any) -> str: - return json.dumps(obj, indent=2) - - -COLLABORATORS_QUERY = """ -query ($owner: String!, $name: String!, $user: String!) { - repository(owner: $owner, name: $name) { - collaborators(query: $user, first: 100) { - nodes { - login - } - } - } -} -""" - -MENTIONABLE_QUERY = """ -query ($owner: String!, $name: String!, $user: String!) { - repository(owner: $owner, name: $name) { - mentionableUsers(query: $user, first: 100) { - nodes { - login - } - } - } -} -""" - - -PR_QUERY = """ - query ($owner: String!, $name: String!, $number: Int!) { - repository(owner: $owner, name: $name) { - pullRequest(number: $number) { - title - body - state - author { - login - } - comments(last: 100) { - pageInfo { - hasPreviousPage - } - nodes { - authorAssociation - author { - login - } - id - updatedAt - body - } - } - authorCommits:commits(last:100) { - nodes { - commit { - authors(first:100) { - nodes { - name - email - } - } - } - } - } - commits(last: 1) { - nodes { - commit { - oid - statusCheckRollup { - contexts(first: 100) { - pageInfo { - hasNextPage - } - nodes { - ... on CheckRun { - name - databaseId - checkSuite { - workflowRun { - databaseId - workflow { - name - } - } - } - status - conclusion - url - } - ... on StatusContext { - state - context - targetUrl - } - } - } - } - } - } - } - reviewDecision - reviews(last: 100) { - pageInfo { - hasPreviousPage - } - nodes { - body - updatedAt - url - id - authorCanPushToRepository - commit { - oid - } - author { - login - } - state - } - } - } - } - } - """ - - -def walk(obj, visitor, parent_key=None): - """ - Recursively call 'visitor' on all the children of a dictionary - """ - visitor(obj, parent_key) - if isinstance(obj, dict): - for k, v in obj.items(): - walk(v, visitor, parent_key=k) - elif isinstance(obj, list): - for v in obj: - walk(v, visitor) - - -class PR: - def __init__( - self, - number: int, - owner: str, - repo: str, - dry_run: bool = False, - raw_data: dict[str, Any] | None = None, - ): - self.owner = owner - self.number = number - self.repo_name = repo - self.dry_run = dry_run - self.has_error = False - - if dry_run and raw_data: - # In test mode there is no need to fetch anything - self.raw = raw_data - self.github = None - else: - self.github = GitHubRepo(user=owner, repo=repo, token=os.environ["GITHUB_TOKEN"]) - if os.getenv("DEBUG", "0") == "1": - # For local runs fill in the requested data but cache it for - # later use - cached_path = Path("pr.json") - if not cached_path.exists(): - self.raw = self.fetch_data() - with open(cached_path, "w") as f: - json.dump(self.raw, f, indent=2) - else: - with open(cached_path) as f: - self.raw = json.load(f) - else: - # Usual path, fetch the PR's data based on the number from - # GitHub - self.raw = self.fetch_data() - - def checker(obj, parent_key): - """ - Verify that any paged results don't have extra data (if so the bot - may still work since most relevant comments will be more recent) - """ - if parent_key == "pageInfo": - if obj.get("hasPreviousPage", False): - warnings.warn(f"Found {obj} with a previous page, bot may be missing data") - if obj.get("hasNextPage", False): - warnings.warn(f"Found {obj} with a next page, bot may be missing data") - - walk(self.raw, checker) - - logging.info(f"Verified data, running with PR {to_json_str(self.raw)}") - - def __repr__(self): - return json.dumps(self.raw, indent=2) - - def react(self, comment: dict[str, Any], content: str): - """ - React with a thumbs up to a comment - """ - url = f"issues/comments/{comment['id']}/reactions" - data = {"content": content} - if self.dry_run: - logging.info(f"Dry run, would have +1'ed to {url} with {data}") - else: - self.github.post(url, data=data) - - def head_commit(self): - return self.raw["commits"]["nodes"][0]["commit"] - - def co_authors(self) -> list[str]: - authors = [] - for commit in self.raw["authorCommits"]["nodes"]: - # Co-authors always come after the main author according to the - # GitHub docs, so ignore the first item - for author in commit["commit"]["authors"]["nodes"][1:]: - name = author["name"] - email = author["email"] - authors.append(f"{name} <{email}>") - - return list(set(authors)) - - def head_oid(self): - return self.head_commit()["oid"] - - def ci_jobs(self) -> list[CIJob]: - """ - Get a list of all CI jobs (GitHub Actions and other) in a unified format - """ - jobs = [] - for item in self.head_commit()["statusCheckRollup"]["contexts"]["nodes"]: - if "checkSuite" in item: - # GitHub Actions job, parse separately - status = item["conclusion"] - if status is None: - # If the 'conclusion' isn't filled out the job hasn't - # finished yet - status = "PENDING" - workflow_name = item["checkSuite"]["workflowRun"]["workflow"]["name"] - if workflow_name != "CI": - # Ignore all jobs that aren't in the main.yml workflow (these are mostly - # automation jobs that run on PRs for tagging / reviews) - continue - check_name = item["name"] - jobs.append( - { - "name": f"{workflow_name} / {check_name}", - "url": item["url"], - "status": status.upper(), - } - ) - else: - # GitHub Status (e.g. from Jenkins) - jobs.append( - { - "name": item["context"], - "url": item["targetUrl"], - "status": item["state"].upper(), - } - ) - - logging.info(f"Found CI jobs for {self.head_commit()['oid']} {to_json_str(jobs)}") - return jobs - - def reviews(self) -> list[Review]: - return self.raw["reviews"]["nodes"] - - def head_commit_reviews(self) -> list[Review]: - """ - Find reviews associated with the head commit - """ - commits_to_review_status: dict[str, list[Review]] = {} - - for review in self.reviews(): - if not review["authorCanPushToRepository"]: - # ignore reviews from non-committers - continue - - oid = review["commit"]["oid"] - if oid in commits_to_review_status: - commits_to_review_status[oid].append(review) - else: - commits_to_review_status[oid] = [review] - - # Only use the data for the head commit of the PR - head_reviews = commits_to_review_status.get(self.head_oid(), []) - return head_reviews - - def fetch_data(self): - """ - Fetch the data for this PR from GitHub - """ - return self.github.graphql( - query=PR_QUERY, - variables={ - "owner": self.owner, - "name": self.repo_name, - "number": self.number, - }, - )["data"]["repository"]["pullRequest"] - - def search_collaborator(self, user: str) -> list[dict[str, Any]]: - """ - Query GitHub for collaborators matching 'user' - """ - return self.search_users(user, COLLABORATORS_QUERY)["collaborators"]["nodes"] - - def search_users(self, user: str, query: str) -> list[dict[str, Any]]: - return self.github.graphql( - query=query, - variables={ - "owner": self.owner, - "name": self.repo_name, - "user": user, - }, - )["data"]["repository"] - - def search_mentionable_users(self, user: str) -> list[dict[str, Any]]: - return self.search_users(user, MENTIONABLE_QUERY)["mentionableUsers"]["nodes"] - - def comment(self, text: str) -> None: - """ - Leave the comment 'text' on this PR - """ - logging.info(f"Commenting:\n{text}") - # TODO: Update latest comment in-place if there has been no activity - data = {"body": text} - url = f"issues/{self.number}/comments" - if self.dry_run: - logging.info( - f"Dry run, would have commented on url={url} commenting with data={to_json_str(data)}" - ) - return - - self.github.post(url, data=data) - - def state(self) -> str: - """ - PR state (OPEN, CLOSED, MERGED, etc) - """ - return self.raw["state"] - - def processed_body(self) -> str: - body = self.raw["body"].strip().replace("\r", "") - # Remove any @-mentions of people - body = re.sub(r"(\s)@", r"\g<1>", body) - - # Remove the auto-inserted text since it's not useful to have in the commit log - body = re.sub(THANKS_MESSAGE, "\n\n", body) - return body.strip() - - def body_with_co_authors(self) -> str: - """ - Add 'Co-authored-by' strings to the PR body based on the prior commits - in the PR - """ - body = self.processed_body() - author_lines = self.co_authors() - logging.info(f"Found co-authors: author_lines={author_lines}") - full_author_lines = [f"Co-authored-by: {author_line}" for author_line in author_lines] - - authors_to_add = [] - for author_line in author_lines: - if author_line not in body: - authors_to_add.append(f"Co-authored-by: {author_line}") - - if len(authors_to_add) > 0: - # If the line isn't already in the PR body (it could have been - # added manually), put it in - full_author_text = "\n".join(authors_to_add) - body = f"{body}\n\n{full_author_text}" - - return body - - def merge(self) -> None: - """ - Request a merge of this PR via the GitHub API - """ - url = f"pulls/{self.number}/merge" - - title = self.raw["title"] + f" (#{self.number})" - body = self.body_with_co_authors() - logging.info(f"Full commit:\n{title}\n\n{body}") - - data = { - "commit_title": title, - "commit_message": body, - # The SHA is necessary in case there was an update right when this - # script ran, GitHub will sort out who won - "sha": self.head_oid(), - "merge_method": "squash", - } - if self.dry_run: - logging.info(f"Dry run, would have merged with url={url} and data={to_json_str(data)}") - return - - r = self.github.put(url, data=data) - logging.info(f"GitHub merge response: {r}") - return r - - def author(self) -> str: - return self.raw["author"]["login"] - - def find_failed_ci_jobs(self) -> list[CIJob]: - # NEUTRAL is GitHub Action's way of saying cancelled - return [ - job - for job in self.ci_jobs() - if job["status"] not in {"SUCCESS", "SUCCESSFUL", "SKIPPED"} - ] - - def find_missing_expected_jobs(self) -> list[str]: - # Map of job name: has seen in completed jobs - seen_expected_jobs = {name: False for name in EXPECTED_JOBS} - logging.info(f"Expected to see jobs: {seen_expected_jobs}") - - missing_expected_jobs = [] - for job in self.ci_jobs(): - seen_expected_jobs[job["name"]] = True - - for name, seen in seen_expected_jobs.items(): - if not seen: - missing_expected_jobs.append(name) - - return missing_expected_jobs - - def trigger_gha_ci(self, sha: str) -> None: - logging.info("POST-ing a workflow_dispatch event to main.yml") - actions_github = GitHubRepo( - user=self.github.user, repo=self.github.repo, token=GH_ACTIONS_TOKEN - ) - r = actions_github.post( - url="actions/workflows/main.yml/dispatches", - data={ - "ref": "main", - }, - ) - logging.info(f"Successful workflow_dispatch: {r}") - - def merge_if_passed_checks(self) -> dict[str, Any] | None: - failed_ci_jobs = self.find_failed_ci_jobs() - all_ci_passed = len(failed_ci_jobs) == 0 - has_one_approval = False - - if not all_ci_passed: - failed_jobs_msg = "\n".join( - [f" * [{job['name']} (`{job['status']}`)]({job['url']})" for job in failed_ci_jobs] - ) - self.comment( - f"Cannot merge, these CI jobs are not successful on {self.head_oid()}:\n{failed_jobs_msg}" - ) - return None - - missing_expected_jobs = self.find_missing_expected_jobs() - - if len(missing_expected_jobs) > 0: - missing_jobs_msg = "\n".join([f" * `{name}`" for name in missing_expected_jobs]) - self.comment(f"Cannot merge, missing expected jobs:\n{missing_jobs_msg}") - return None - - head_commit_reviews = self.head_commit_reviews() - for review in head_commit_reviews: - if review["state"] == "CHANGES_REQUESTED": - self.comment( - f"Cannot merge, found [this review]({review['url']}) on {self.head_oid()} with changes requested" - ) - return None - - if review["state"] == "APPROVED": - has_one_approval = True - logging.info(f"Found approving review: {to_json_str(review)}") - - if has_one_approval and all_ci_passed: - return self.merge() - elif not has_one_approval: - self.comment( - f"Cannot merge, did not find any approving reviews from users with write access on {self.head_oid()}" - ) - return None - elif not all_ci_passed: - self.comment(f"Cannot merge, CI did not pass on on {self.head_oid()}") - return None - - def rerun_jenkins_ci(self) -> None: - job_names = [ - "tvm-arm", - "tvm-cpu", - "tvm-docker", - "tvm-gpu", - "tvm-wasm", - ] - for name in job_names: - url = JENKINS_URL + f"job/{name}/job/PR-{self.number}/buildWithParameters" - logging.info(f"Rerunning ci with URL={url}") - if self.dry_run: - logging.info("Dry run, not sending POST") - else: - post(url, auth=("tvm-bot", TVM_BOT_JENKINS_TOKEN)) - - def rerun_github_actions(self) -> None: - workflow_ids = [] - for item in self.head_commit()["statusCheckRollup"]["contexts"]["nodes"]: - if "checkSuite" in item and item["conclusion"] == "FAILURE": - workflow_id = item["checkSuite"]["workflowRun"]["databaseId"] - workflow_ids.append(workflow_id) - - workflow_ids = list(set(workflow_ids)) - logging.info(f"Rerunning GitHub Actions workflows with IDs: {workflow_ids}") - if self.dry_run: - actions_github = None - else: - actions_github = GitHubRepo( - user=self.github.user, repo=self.github.repo, token=GH_ACTIONS_TOKEN - ) - for workflow_id in workflow_ids: - if self.dry_run: - logging.info(f"Dry run, not restarting workflow {workflow_id}") - else: - try: - actions_github.post(f"actions/runs/{workflow_id}/rerun-failed-jobs", data={}) - except RuntimeError as e: - logging.exception(e) - # Ignore errors about jobs that are part of the same workflow to avoid - # having to figure out which jobs are in which workflows ahead of time - if "The workflow run containing this job is already running" in str(e): - pass - else: - raise e - - def comment_failure(self, msg: str, exceptions: Exception | list[Exception]): - if not isinstance(exceptions, list): - exceptions = [exceptions] - - logging.info(f"Failed, commenting {exceptions}") - - # Extract all the traceback strings - for item in exceptions: - try: - raise item - except Exception: - item.exception_msg = traceback.format_exc() - - comment = f"{msg} in {args.run_url}\n\n" - for exception in exceptions: - comment += f"<details>\n\n```\n{exception.exception_msg}\n```\n\n" - if hasattr(exception, "read"): - comment += f"with response\n\n```\n{exception.read().decode()}\n```\n\n" - comment += "</details>" - - pr.comment(comment) - pr.has_error = True - return exception - - -def check_author(pr, triggering_comment, args): - comment_author = triggering_comment["user"]["login"] - if pr.author() == comment_author: - logging.info("Comment user is PR author, continuing") - return True - return False - - -def search_users(name, triggering_comment, testing_json, search_fn): - logging.info(f"Checking {name}") - commment_author = triggering_comment["user"]["login"] - if testing_json: - matching_users = json.loads(testing_json) - else: - matching_users = search_fn(commment_author) - logging.info(f"Found {name}: {matching_users}") - user_names = {user["login"] for user in matching_users} - - return len(matching_users) > 0 and commment_author in user_names - - -def check_collaborator(pr, triggering_comment, args): - return search_users( - name="collaborators", - triggering_comment=triggering_comment, - search_fn=pr.search_collaborator, - testing_json=args.testing_collaborators_json, - ) - - -def check_mentionable_users(pr, triggering_comment, args): - return search_users( - name="mentionable users", - triggering_comment=triggering_comment, - search_fn=pr.search_mentionable_users, - testing_json=args.testing_mentionable_users_json, - ) - - -AUTH_CHECKS = { - "mentionable_users": check_mentionable_users, - "collaborators": check_collaborator, - "author": check_author, -} -# Stash the keys so they're accessible from the values -AUTH_CHECKS = {k: (k, v) for k, v in AUTH_CHECKS.items()} - - -class Merge: - triggers = [ - "merge", - "merge this", - "merge this pr", - ] - - auth = [AUTH_CHECKS["collaborators"], AUTH_CHECKS["author"]] - - @staticmethod - def run(pr: PR): - info = None - try: - info = pr.merge_if_passed_checks() - except Exception as e: - pr.comment_failure("Failed to process merge request", e) - raise e - - if info is not None: - try: - pr.trigger_gha_ci(sha=info["sha"]) - except Exception as e: - pr.comment_failure("Failed to trigger GitHub Actions", e) - raise e - - -class Rerun: - triggers = [ - "rerun", - "rerun ci", - "re-run", - "re-run ci", - "run", - "run ci", - ] - - auth = [AUTH_CHECKS["mentionable_users"]] - - @staticmethod - def run(pr: PR): - errors = [] - try: - pr.rerun_jenkins_ci() - except Exception as e: - logging.exception(e) - errors.append(e) - - try: - pr.rerun_github_actions() - except Exception as e: - logging.exception(e) - errors.append(e) - - if len(errors) > 0: - pr.comment_failure("Failed to re-run CI", errors) - - -if __name__ == "__main__": - help = "Check if a PR has comments trying to merge it, and do so based on reviews/CI status" - parser = argparse.ArgumentParser(description=help) - parser.add_argument("--remote", default="origin", help="ssh remote to parse") - parser.add_argument("--pr", required=True, help="pr number to check") - parser.add_argument("--run-url", required=True, help="workflow run URL") - parser.add_argument( - "--trigger-comment-json", required=True, help="json of the comment that triggered this run" - ) - parser.add_argument("--testing-pr-json", help="(testing only) manual data for testing") - parser.add_argument( - "--testing-collaborators-json", help="(testing only) manual data for testing" - ) - parser.add_argument( - "--testing-mentionable-users-json", help="(testing only) manual data for testing" - ) - parser.add_argument( - "--dry-run", - action="store_true", - default=False, - help="run but don't send any request to GitHub", - ) - args = parser.parse_args() - init_log() - comment = json.loads(args.trigger_comment_json) - body = comment["body"].strip() - - # Check that the comment was addressed to tvm-bot - if not body.startswith("@tvm-bot "): - logging.info(f"Not a bot comment, '{body}' does not start with '@tvm-bot'") - exit(0) - - # Find the code to run for the command from the user - user_command = body.lstrip("@tvm-bot").strip() - command_to_run = None - for command in [Merge, Rerun]: - if user_command in command.triggers: - command_to_run = command - break - - if command_to_run is None: - logging.info(f"Command '{user_command}' did not match anything") - exit(0) - - # Find the remote for querying more data about the PR - remote = git(["config", "--get", f"remote.{args.remote}.url"]) - logging.info(f"Using remote remote={remote}") - owner, repo = parse_remote(remote) - - if args.pr.strip() == "": - logging.info("No PR number passed") - exit(0) - - logging.info(f"Checking owner={owner} repo={repo}") - if args.testing_pr_json: - pr = PR( - number=int(args.pr), - owner=owner, - repo=repo, - dry_run=args.dry_run, - raw_data=json.loads(args.testing_pr_json), - ) - else: - pr = PR(number=int(args.pr), owner=owner, repo=repo, dry_run=args.dry_run) - - for name, check in command_to_run.auth: - if check(pr, comment, args): - logging.info(f"Passed auth check '{name}', continuing") - # Only one authorization check needs to pass (e.g. just mentionable - # or PR author), not all of them so quit - break - else: - logging.info(f"Failed auth check '{name}', quitting") - # Add a sad face - pr.react(comment, "confused") - exit(0) - - # Acknowledge the comment with a react - pr.react(comment, "+1") - - state = pr.state() - - if state != "OPEN": - logging.info(f"Ignoring event on PR, state was not OPEN, instead was state={state}") - exit(0) - - # Run the command - command_to_run.run(pr) - - if pr.has_error: - raise RuntimeError("PR commented a failure") diff --git a/ci/scripts/jenkins/cmd_utils.py b/ci/scripts/jenkins/cmd_utils.py deleted file mode 100644 index e1cd8e9b20..0000000000 --- a/ci/scripts/jenkins/cmd_utils.py +++ /dev/null @@ -1,107 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging -import os -import re -import subprocess -import sys -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -assert (REPO_ROOT / "Jenkinsfile").exists - - -class RelativePathFilter(logging.Filter): - def filter(self, record): - path = Path(record.pathname).resolve() - record.relativepath = str(path.relative_to(REPO_ROOT)) - return True - - -def init_log(): - logging.basicConfig( - format="[%(relativepath)s:%(lineno)d %(levelname)-1s] %(message)s", level=logging.INFO - ) - - # Flush on every log call (logging and then calling subprocess.run can make - # the output look confusing) - logging.root.handlers[0].addFilter(RelativePathFilter()) - logging.root.handlers[0].flush = sys.stderr.flush - - -class Sh: - def __init__(self, env=None, cwd=None): - self.env = os.environ.copy() - if env is not None: - self.env.update(env) - self.cwd = cwd - - def tee(self, cmd: str, **kwargs): - """ - Run 'cmd' in a shell then return the (process, stdout) as a tuple - """ - - logging.info(f"+ {cmd}") - - kwargs = { - **self._default_popen_flags(), - **kwargs, - "stdout": subprocess.PIPE, - } - proc = subprocess.Popen(cmd, **kwargs) - - stdout = [] - - def _tee_output(s): - stdout.append(s) - print(s, end="") - - while proc.poll() is None: - _tee_output(proc.stdout.readline()) - _tee_output(proc.stdout.read()) - - stdout = "".join(stdout) - if proc.returncode: - raise subprocess.CalledProcessError(proc.returncode, proc.args, stdout) - - return proc, stdout - - def run(self, cmd: str, **kwargs): - logging.info(f"+ {cmd}") - - kwargs = { - **self._default_popen_flags(), - "check": True, - **kwargs, - } - - return subprocess.run(cmd, **kwargs) - - def _default_popen_flags(self): - return { - "shell": True, - "env": self.env, - "encoding": "utf-8", - "cwd": self.cwd, - } - - -def tags_from_title(title: str) -> list[str]: - tags = re.findall(r"\[(.*?)\]", title) - tags = [t.strip() for t in tags] - return tags diff --git a/ci/scripts/jenkins/determine_docker_images.py b/ci/scripts/jenkins/determine_docker_images.py deleted file mode 100755 index 3f3deb5c57..0000000000 --- a/ci/scripts/jenkins/determine_docker_images.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import argparse -import configparser -import datetime -import json -import logging -import urllib.error -from pathlib import Path -from typing import Any - -from cmd_utils import REPO_ROOT, init_log -from http_utils import get - -DOCKER_API_BASE = "https://hub.docker.com/v2/"; -PAGE_SIZE = 25 -TEST_DATA = None -IMAGE_TAGS_FILE = REPO_ROOT / "ci" / "jenkins" / "docker-images.ini" -TVM_CI_ECR = "477529581014.dkr.ecr.us-west-2.amazonaws.com" - - -def docker_api(url: str, use_pagination: bool = False) -> dict[str, Any]: - """ - Run a paginated fetch from the public Docker Hub API - """ - if TEST_DATA is not None: - if url not in TEST_DATA: - raise urllib.error.HTTPError(url, 404, "Not found", {}, None) - return TEST_DATA[url] - pagination = "" - if use_pagination: - pagination = f"?page_size={PAGE_SIZE}&page=1" - url = DOCKER_API_BASE + url + pagination - r, headers = get(url) - reset = headers.get("x-ratelimit-reset") - if reset is not None: - reset = datetime.datetime.fromtimestamp(int(reset)) - reset = reset.isoformat() - logging.info( - f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})" - ) - return r - - -def image_exists(spec: str) -> bool: - name, tag = spec.split(":") - try: - r = docker_api(f"repositories/{name}/tags/{tag}") - logging.debug(f"Image exists, got response: {json.dumps(r, indent=2)}") - return True - except urllib.error.HTTPError as e: - # Image was not found - logging.debug(e) - return False - - -def lookup_image_tag(name: str) -> str: - """Resolve image ``name`` (e.g. ``ci_cpu``) to its tag string from the ini. - - Pure ini read — no Docker Hub query, no tlcpackstaging fallback. Used by - ``docker/dev_common.sh`` for local-dev image-name shortcuts where the - full Hub-existence check is unnecessary. - """ - config = configparser.ConfigParser() - config.read(IMAGE_TAGS_FILE) - return config.get("jenkins", name) - - -if __name__ == "__main__": - init_log() - parser = argparse.ArgumentParser( - description="Writes out Docker images names to be used to .docker-image-names/" - ) - parser.add_argument( - "--lookup-only", - metavar="NAME", - help="Print the tag for NAME from the ini and exit (no Docker Hub query, no fallback).", - ) - parser.add_argument( - "--testing-docker-data", - help="(testing only) JSON data to mock response from Docker Hub API", - ) - parser.add_argument( - "--testing-images-data", - help=f"(testing only) JSON data to mock contents of {IMAGE_TAGS_FILE}", - ) - parser.add_argument( - "--base-dir", - default=".docker-image-names", - help="(testing only) Folder to write image names to", - ) - args, other = parser.parse_known_args() - - if args.lookup_only: - print(lookup_image_tag(args.lookup_only)) - raise SystemExit(0) - - name_dir = Path(args.base_dir) - - if args.testing_images_data: - repo_image_tags = json.loads(args.testing_images_data) - else: - config = configparser.ConfigParser() - config.read(IMAGE_TAGS_FILE) - repo_image_tags = {} - for name in other: - repo_image_tags[name] = config.get("jenkins", name) - - images = {} - for name in other: - images[name] = repo_image_tags[name] - - if args.testing_docker_data is not None: - TEST_DATA = json.loads(args.testing_docker_data) - - logging.info(f"Checking if these images exist in tlcpack: {images}") - - name_dir.mkdir(exist_ok=True) - images_to_use = {} - for filename, spec in images.items(): - if spec.startswith(TVM_CI_ECR): - logging.info(f"{spec} is from ECR") - images_to_use[filename] = spec - elif image_exists(spec): - logging.info(f"{spec} found in tlcpack") - images_to_use[filename] = spec - else: - logging.info(f"{spec} not found in tlcpack, using tlcpackstaging") - part, tag = spec.split(":") - user, repo = part.split("/") - tlcpackstaging_tag = f"tlcpackstaging/{repo.replace('-', '_')}:{tag}" - images_to_use[filename] = tlcpackstaging_tag - - for filename, image in images_to_use.items(): - logging.info(f"Writing image {image} to {name_dir / filename}") - with open(name_dir / filename, "w") as f: - f.write(image) diff --git a/ci/scripts/jenkins/git_change_docker.sh b/ci/scripts/jenkins/git_change_docker.sh deleted file mode 100755 index 758ccfdf6e..0000000000 --- a/ci/scripts/jenkins/git_change_docker.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -if [ "${BRANCH_NAME}" == "main" ]; then - changed_files=$(git diff --no-commit-id --name-only -r HEAD~1) -else - changed_files=$(git diff --no-commit-id --name-only -r origin/main) -fi - -FILES_THAT_SHOULDNT_TRIGGER_REBUILDS=( - "docker/bash.sh" - "docker/with_the_same_user" - "docker/README.md" - "docker/Dockerfile.ci_lint" - "docker/install/ubuntu_install_clang_format.sh" - "docker/lint.sh" - "docker/clear-stale-images.sh" - "README.md" - "lint.sh" - "clear-stale-images.sh" -) - -for file in $changed_files; do - # Certain files under docker/ don't matter for rebuilds, so ignore them - if printf '%s\0' "${FILES_THAT_SHOULDNT_TRIGGER_REBUILDS[@]}" | grep -F -x -z -- "$file"; then - echo "Skipping $file" - continue - fi - # if grep -q "docker/" - echo "Checking $file" - if grep -q "docker/" <<< "$file"; then - exit 1 - fi -done - -# No docker changes -exit 0 diff --git a/ci/scripts/jenkins/git_change_docs.sh b/ci/scripts/jenkins/git_change_docs.sh deleted file mode 100755 index 35b1bb7ec3..0000000000 --- a/ci/scripts/jenkins/git_change_docs.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -FOUND_ONE_FILE=0 -SAW_NON_DOC_CHANGES=0 - -changed_files=$(git diff --no-commit-id --name-only -r origin/main) - -for file in $changed_files; do - FOUND_ONE_FILE=1 - if ! grep -q "docs/" <<< "$file"; then - SAW_NON_DOC_CHANGES=1 - break - fi -done - -if [ ${FOUND_ONE_FILE} -eq 0 ] || [ ${SAW_NON_DOC_CHANGES} -eq 1 ]; then - exit 0 -else - exit 1 -fi diff --git a/ci/scripts/jenkins/git_skip_ci.py b/ci/scripts/jenkins/git_skip_ci.py deleted file mode 100755 index 3f7caa0f8d..0000000000 --- a/ci/scripts/jenkins/git_skip_ci.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import argparse -import logging -import os - -from cmd_utils import init_log, tags_from_title -from git_utils import GitHubRepo, git, parse_remote - -if __name__ == "__main__": - help = "Exits with 0 if CI should be skipped, 1 otherwise" - parser = argparse.ArgumentParser(description=help) - parser.add_argument("--pr", required=True) - parser.add_argument("--remote", default="origin", help="ssh remote to parse") - parser.add_argument( - "--pr-title", help="(testing) PR title to use instead of fetching from GitHub" - ) - args = parser.parse_args() - init_log() - - branch = git(["rev-parse", "--abbrev-ref", "HEAD"]) - log = git(["log", "--format=%s", "-1"]) - - # Check the PR's title (don't check this until everything else passes first) - def check_pr_title(): - remote = git(["config", "--get", f"remote.{args.remote}.url"]) - user, repo = parse_remote(remote) - - if args.pr_title: - title = args.pr_title - else: - github = GitHubRepo(token=os.environ["GITHUB_TOKEN"], user=user, repo=repo) - pr = github.get(f"pulls/{args.pr}") - title = pr["title"] - logging.info(f"pr title: {title}") - tags = tags_from_title(title) - logging.info(f"Found title tags: {tags}") - return "skip ci" in tags - - if args.pr != "null" and args.pr.strip() != "" and branch != "main" and check_pr_title(): - logging.info("PR title starts with '[skip ci]', skipping...") - exit(0) - else: - logging.info(f"Not skipping CI:\nargs.pr: {args.pr}\nbranch: {branch}\ncommit: {log}") - exit(1) diff --git a/ci/scripts/jenkins/git_skip_ci_globs.py b/ci/scripts/jenkins/git_skip_ci_globs.py deleted file mode 100755 index 90970eecc6..0000000000 --- a/ci/scripts/jenkins/git_skip_ci_globs.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 - -import argparse -import fnmatch - -from git_utils import git - -globs = [ - "*.md", - ".github/*", - ".asf.yaml", - ".gitignore", - "LICENSE", - "NOTICE", - "KEYS", - "tests/lint/*", - ".pre-commit-config.yaml", -] - - -def match_any(f: str) -> str | None: - for glob in globs: - if fnmatch.fnmatch(f, glob): - return glob - return None - - -if __name__ == "__main__": - help = "Exits with code 1 if a change only touched files, indicating that CI could be skipped for this changeset" - parser = argparse.ArgumentParser(description=help) - parser.add_argument("--files", help="(testing only) comma separated list of files to check") - args = parser.parse_args() - print(args) - if args.files is not None: - diff = [x for x in args.files.split(",") if x.strip() != ""] - else: - diff = git(["diff", "--no-commit-id", "--name-only", "-r", "origin/main"]) - diff = diff.split("\n") - diff = [d.strip() for d in diff] - diff = [d for d in diff if d != ""] - - print(f"Changed files:\n{diff}") - - if len(diff) == 0: - print("Found no changed files, skipping CI") - exit(0) - - print(f"Checking with globs:\n{globs}") - - for file in diff: - match = match_any(file) - if match is None: - print(f"{file} did not match any globs, running CI") - exit(1) - else: - print(f"{file} matched glob {match}") - - print("All files matched a glob, skipping CI") - exit(0) diff --git a/ci/scripts/jenkins/git_utils.py b/ci/scripts/jenkins/git_utils.py deleted file mode 100644 index b16e3f7aa3..0000000000 --- a/ci/scripts/jenkins/git_utils.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501, RUF005 - -import base64 -import json -import logging -import os -import re -import subprocess -from typing import Any -from urllib import error, request - -DRY_RUN = object() - - -def compress_query(query: str) -> str: - query = query.replace("\n", "") - query = re.sub("\s+", " ", query) - return query - - -def post(url: str, body: Any | None = None, auth: tuple[str, str] | None = None): - logging.info(f"Requesting POST to {url} with {body}") - headers = {} - req = request.Request(url, headers=headers, method="POST") - if auth is not None: - auth_str = base64.b64encode(f"{auth[0]}:{auth[1]}".encode()) - req.add_header("Authorization", f"Basic {auth_str.decode()}") - - if body is None: - body = "" - - req.add_header("Content-Type", "application/json; charset=utf-8") - data = json.dumps(body) - data = data.encode("utf-8") - req.add_header("Content-Length", len(data)) - - with request.urlopen(req, data) as response: - return response.read() - - -def dry_run_token(is_dry_run: bool) -> Any: - if is_dry_run: - return DRY_RUN - return os.environ["GITHUB_TOKEN"] - - -class GitHubRepo: - GRAPHQL_URL = "https://api.github.com/graphql"; - - def __init__(self, user, repo, token, test_data=None): - self.token = token - self.user = user - self.repo = repo - self.test_data = test_data - self.num_calls = 0 - self.base = f"https://api.github.com/repos/{user}/{repo}/"; - - def headers(self): - return { - "Authorization": f"Bearer {self.token}", - } - - def dry_run(self) -> bool: - return self.token == DRY_RUN - - def graphql(self, query: str, variables: dict[str, str] | None = None) -> dict[str, Any]: - query = compress_query(query) - if variables is None: - variables = {} - - response = self._request( - self.GRAPHQL_URL, - {"query": query, "variables": variables}, - method="POST", - ) - if self.dry_run(): - return self.testing_response("POST", self.GRAPHQL_URL) - - if "data" not in response: - msg = f"Error fetching data with query:\n{query}\n\nvariables:\n{variables}\n\nerror:\n{json.dumps(response, indent=2)}" - raise RuntimeError(msg) - return response - - def testing_response(self, method: str, url: str) -> Any: - self.num_calls += 1 - key = f"[{self.num_calls}] {method} - {url}" - if self.test_data is not None and key in self.test_data: - return self.test_data[key] - logging.info(f"Unknown URL in dry run: {key}") - return {} - - def _request(self, full_url: str, body: dict[str, Any], method: str) -> dict[str, Any]: - if self.dry_run(): - logging.info(f"Dry run, would have requested a {method} to {full_url} with {body}") - return self.testing_response(method, full_url) - - logging.info(f"Requesting {method} to {full_url} with {body}") - req = request.Request(full_url, headers=self.headers(), method=method.upper()) - req.add_header("Content-Type", "application/json; charset=utf-8") - data = json.dumps(body) - data = data.encode("utf-8") - req.add_header("Content-Length", len(data)) - - try: - with request.urlopen(req, data) as response: - content = response.read() - except error.HTTPError as e: - msg = str(e) - error_data = e.read().decode() - raise RuntimeError(f"Error response: {msg}\n{error_data}") - - logging.info(f"Got response from {full_url}: {content}") - try: - response = json.loads(content) - except json.decoder.JSONDecodeError: - return content - - return response - - def put(self, url: str, data: dict[str, Any]) -> dict[str, Any]: - return self._request(self.base + url, data, method="PUT") - - def patch(self, url: str, data: dict[str, Any]) -> dict[str, Any]: - return self._request(self.base + url, data, method="PATCH") - - def post(self, url: str, data: dict[str, Any]) -> dict[str, Any]: - return self._request(self.base + url, data, method="POST") - - def get(self, url: str) -> dict[str, Any]: - if self.dry_run(): - logging.info(f"Dry run, would have requested a GET to {url}") - return self.testing_response("GET", url) - url = self.base + url - logging.info(f"Requesting GET to {url}") - req = request.Request(url, headers=self.headers()) - with request.urlopen(req) as response: - response = json.loads(response.read()) - return response - - def delete(self, url: str) -> dict[str, Any]: - if self.dry_run(): - logging.info(f"Dry run, would have requested a DELETE to {url}") - return self.testing_response("DELETE", url) - url = self.base + url - logging.info(f"Requesting DELETE to {url}") - req = request.Request(url, headers=self.headers(), method="DELETE") - with request.urlopen(req) as response: - response = json.loads(response.read()) - return response - - -def parse_remote(remote: str) -> tuple[str, str]: - """ - Get a GitHub (user, repo) pair out of a git remote - """ - if remote.startswith("https://";): - # Parse HTTP remote - parts = remote.split("/") - if len(parts) < 2: - raise RuntimeError(f"Unable to parse remote '{remote}'") - user, repo = parts[-2], parts[-1].replace(".git", "") - else: - # Parse SSH remote - m = re.search(r":(.*)/(.*)\.git", remote) - if m is None or len(m.groups()) != 2: - raise RuntimeError(f"Unable to parse remote '{remote}'") - user, repo = m.groups() - - user = os.getenv("DEBUG_USER", user) - repo = os.getenv("DEBUG_REPO", repo) - return user, repo - - -def git(command, **kwargs): - command = ["git"] + command - logging.info(f"Running {command}") - proc = subprocess.run(command, stdout=subprocess.PIPE, encoding="utf-8", **kwargs) - if proc.returncode != 0: - raise RuntimeError(f"Command failed {command}:\nstdout:\n{proc.stdout}") - return proc.stdout.strip() - - -def find_ccs(body: str) -> list[str]: - matches = re.findall(r"(cc( @[-A-Za-z0-9]+)+)", body, flags=re.MULTILINE) - matches = [full for full, last in matches] - - reviewers = [] - for match in matches: - if match.startswith("cc "): - match = match.replace("cc ", "") - users = [x.strip() for x in match.split("@")] - reviewers += users - - reviewers = set(x for x in reviewers if x != "") - return list(reviewers) diff --git a/ci/scripts/jenkins/http_utils.py b/ci/scripts/jenkins/http_utils.py deleted file mode 100644 index 05e1b71f24..0000000000 --- a/ci/scripts/jenkins/http_utils.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json -import logging -from typing import Any -from urllib import request - - -def get(url: str, headers: dict[str, str] | None = None) -> dict[str, Any]: - logging.info(f"Requesting GET to {url}") - if headers is None: - headers = {} - req = request.Request(url, headers=headers) - with request.urlopen(req) as response: - response_headers = {k: v for k, v in response.getheaders()} - response = json.loads(response.read()) - - return response, response_headers diff --git a/ci/scripts/jenkins/retry.sh b/ci/scripts/jenkins/retry.sh deleted file mode 100644 index 933eb1aba6..0000000000 --- a/ci/scripts/jenkins/retry.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -retry() { - local max_retries=$1 - shift - local n=0 - until [ "$n" -ge "$max_retries" ] - do - "$@" && break - n=$((n+1)) - if [ "$n" -eq "$max_retries" ]; then - echo "failed to update after attempt $n / $max_retries, giving up" - exit 1 - fi - - WAIT=$(python3 -c 'import random; print(random.randint(30, 200))') - echo "failed to update $n / $max_retries, waiting $WAIT to try again" - sleep "$WAIT" - done -} diff --git a/ci/scripts/jenkins/s3.py b/ci/scripts/jenkins/s3.py deleted file mode 100755 index e2e65dcadb..0000000000 --- a/ci/scripts/jenkins/s3.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import argparse -import importlib.util -import logging -import re -from enum import Enum -from pathlib import Path - -from cmd_utils import REPO_ROOT, Sh, init_log - -DATA_PY = REPO_ROOT / "ci" / "jenkins" / "data.py" - - -def load_files_to_stash(): - """Load the files_to_stash dict from ci/jenkins/data.py.""" - spec = importlib.util.spec_from_file_location("data", DATA_PY) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod.files_to_stash - - -def resolve_bundles(bundle_names: list[str]) -> list[str]: - """Resolve a list of bundle names to a flat list of file paths.""" - files_to_stash = load_files_to_stash() - items = [] - for name in bundle_names: - if name not in files_to_stash: - known = list(files_to_stash.keys()) - logging.error(f"Unknown bundle '{name}'. Known bundles: {known}") - raise SystemExit(1) - items.extend(files_to_stash[name]) - return items - - -RETRY_SCRIPT = REPO_ROOT / "ci" / "scripts" / "jenkins" / "retry.sh" -S3_DOWNLOAD_REGEX = re.compile(r"download: s3://.* to (.*)") -SH = Sh() - - -class Action(Enum): - UPLOAD = 1 - DOWNLOAD = 2 - - -def show_md5(item: str) -> None: - if not Path(item).is_dir(): - sh.run(f"md5sum {item}") - - -def parse_output_files(stdout: str) -> list[str]: - """ - Grab the list of downloaded files from the output of 'aws s3 cp'. Lines look - like: - - download: s3://some/prefix/a_file.txt to a_file.txt - """ - files = [] - for line in stdout.split("\n"): - line = line.strip() - if line == "": - continue - m = S3_DOWNLOAD_REGEX.match(line) - if m: - files.append(m.groups()[0]) - - return files - - -def chmod(files: list[str]) -> None: - """ - S3 has no concept of file permissions so add them back in here to every file - """ - # Add execute bit for downloads - to_chmod = [str(f) for f in files] - logging.info(f"Adding execute bit for files: {to_chmod}") - if len(to_chmod) > 0: - SH.run(f"chmod +x {' '.join(to_chmod)}") - - -def s3(source: str, destination: str, recursive: bool) -> list[str]: - """ - Send or download the source to the destination in S3 - """ - cmd = f". {RETRY_SCRIPT.relative_to(REPO_ROOT)} && retry 3 aws s3 cp --no-progress" - - if recursive: - cmd += " --recursive" - - cmd += f" {source} {destination}" - _, stdout = SH.tee(cmd) - return stdout - - -if __name__ == "__main__": - init_log() - help = "Uploads or downloads files from S3" - parser = argparse.ArgumentParser(description=help) - parser.add_argument("--action", help="either 'upload' or 'download'", required=True) - parser.add_argument("--bucket", help="s3 bucket", required=True) - parser.add_argument( - "--prefix", help="s3 bucket + tag (e.g. s3://tvm-ci-prod/PR-1234/cpu", required=True - ) - parser.add_argument("--items", help="files and folders to upload", nargs="+") - parser.add_argument( - "--bundle", - help="bundle name(s) from ci/jenkins/data.py files_to_stash (repeatable)", - action="append", - dest="bundles", - metavar="NAME", - ) - - args = parser.parse_args() - - if args.items is not None and args.bundles is not None: - parser.error("--items and --bundle are mutually exclusive") - - logging.info(args) - - sh = Sh() - - if Path.cwd() != REPO_ROOT: - logging.error(f"s3.py can only be executed from the repo root, instead was in {Path.cwd()}") - exit(1) - - prefix = args.prefix.strip("/") - s3_path = f"s3://{args.bucket}/{prefix}" - logging.info(f"Using s3 path: {s3_path}") - - if args.action == "upload": - action = Action.UPLOAD - elif args.action == "download": - action = Action.DOWNLOAD - else: - logging.error(f"Unsupported action: {args.action}") - exit(1) - - if args.bundles is not None: - items = resolve_bundles(args.bundles) - elif args.items is not None: - items = args.items - else: - if args.action == "upload": - logging.error("Cannot upload without --items or --bundle") - exit(1) - else: - # Download the whole prefix - items = ["."] - - for item in items: - if action == Action.DOWNLOAD: - source = s3_path - recursive = True - if item != ".": - source = s3_path + "/" + item - recursive = False - try: - stdout = s3(source=source, destination=item, recursive=recursive) - except Exception: - # Optional artifacts (e.g. per-backend device runtime DSOs) may not - # exist in S3 when the build config didn't produce them. Skip silently. - logging.warning(f"Download failed for {item}, skipping (may be optional)") - continue - files = parse_output_files(stdout) - chmod(files) - for file in files: - # Show md5 after downloading - show_md5(file) - elif action == Action.UPLOAD: - if not Path(item).exists(): - logging.warning(f"The path doesn't exist: {item}") - continue - show_md5(item) - if Path(item).is_dir(): - if len(list(Path(item).glob("**/*"))) == 0: - raise RuntimeError(f"Cannot upload empty folder with name: {item}") - s3(item, s3_path + "/" + item, recursive=Path(item).is_dir()) diff --git a/ci/scripts/jenkins/should_rebuild_docker.py b/ci/scripts/jenkins/should_rebuild_docker.py deleted file mode 100755 index 0c608dc43a..0000000000 --- a/ci/scripts/jenkins/should_rebuild_docker.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import argparse -import datetime -import json -import logging -import subprocess -from typing import Any - -from cmd_utils import Sh, init_log -from http_utils import get - -DOCKER_API_BASE = "https://hub.docker.com/v2/"; -PAGE_SIZE = 25 -TEST_DATA = None - - -def docker_api(url: str) -> dict[str, Any]: - """ - Run a paginated fetch from the public Docker Hub API - """ - if TEST_DATA is not None: - return TEST_DATA[url] - pagination = f"?page_size={PAGE_SIZE}&page=1" - url = DOCKER_API_BASE + url + pagination - r, headers = get(url) - reset = headers.get("x-ratelimit-reset") - if reset is not None: - reset = datetime.datetime.fromtimestamp(int(reset)) - reset = reset.isoformat() - logging.info( - f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})" - ) - if "results" not in r: - raise RuntimeError(f"Error fetching data, no results found in: {r}") - return r - - -def any_docker_changes_since(hash: str) -> bool: - """ - Check the docker/ directory, return True if there have been any code changes - since the specified hash - """ - sh = Sh() - cmd = f"git diff {hash} -- docker/" - proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout = proc.stdout.strip() - return stdout != "", stdout - - -def does_commit_exist(hash: str) -> bool: - """ - Returns True if the hash exists in the repo - """ - sh = Sh() - cmd = f"git rev-parse -q {hash}" - proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) - print(proc.stdout) - if proc.returncode == 0: - return True - - if "unknown revision or path not in the working tree" in proc.stdout: - return False - - raise RuntimeError(f"Unexpected failure when running: {cmd}") - - -def find_hash_for_tag(tag: dict[str, Any]) -> str: - """ - Split the hash off of a name like <date>-<time>-<hash> - """ - name = tag["name"] - name_parts = name.split("-") - if len(name_parts) != 3: - raise RuntimeError(f"Image {name} is not using new naming scheme") - shorthash = name_parts[2] - return shorthash - - -def find_commit_in_repo(tags: list[dict[str, Any]]): - """ - Look through all the docker tags, find the most recent one which references - a commit that is present in the repo - """ - for tag in tags["results"]: - shorthash = find_hash_for_tag(tag) - logging.info(f"Hash '{shorthash}' does not exist in repo") - if does_commit_exist(shorthash): - return shorthash, tag - - raise RuntimeError(f"No extant hash found in tags:\n{tags}") - - -def main(): - # Fetch all tlcpack images - images = docker_api("repositories/tlcpack") - - # Ignore all non-ci images - relevant_images = [image for image in images["results"] if image["name"].startswith("ci-")] - image_names = [image["name"] for image in relevant_images] - logging.info(f"Found {len(relevant_images)} images to check: {', '.join(image_names)}") - - for image in relevant_images: - # Check the tags for the image - tags = docker_api(f"repositories/tlcpack/{image['name']}/tags") - - # Find the hash of the most recent tag - shorthash, tag = find_commit_in_repo(tags) - name = tag["name"] - logging.info(f"Looking for docker/ changes since {shorthash}") - - any_docker_changes, diff = any_docker_changes_since(shorthash) - if any_docker_changes: - logging.info(f"Found docker changes from {shorthash} when checking {name}") - logging.info(diff) - exit(2) - - logging.info("Did not find changes, no rebuild necessary") - exit(0) - - -if __name__ == "__main__": - init_log() - parser = argparse.ArgumentParser( - description="Exits 0 if Docker images don't need to be rebuilt, 1 otherwise" - ) - parser.add_argument( - "--testing-docker-data", - help="(testing only) JSON data to mock response from Docker Hub API", - ) - args = parser.parse_args() - - if args.testing_docker_data is not None: - TEST_DATA = json.loads(args.testing_docker_data) - - main() diff --git a/ci/scripts/jenkins/should_run_slow_tests.py b/ci/scripts/jenkins/should_run_slow_tests.py deleted file mode 100755 index 31126d7daf..0000000000 --- a/ci/scripts/jenkins/should_run_slow_tests.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import argparse -import os -import textwrap - -from git_utils import GitHubRepo, git, parse_remote - -SLOW_TEST_TRIGGERS = [ - "@tvm-bot run slow tests", - "@tvm-bot run slow test", - "@tvm-bot run slow", - "@tvm-bot slow tests", - "@tvm-bot slow test", - "@tvm-bot slow", -] - - -def check_match(s: str, searches: list[str]) -> tuple[bool, str | None]: - for search in searches: - if search in s: - return True, search - - return False, None - - -def display(long_str: str) -> str: - return textwrap.indent(long_str, " ") - - -if __name__ == "__main__": - help = "Exits with 1 if CI should run slow tests, 0 otherwise" - parser = argparse.ArgumentParser(description=help) - parser.add_argument("--pr", required=True) - parser.add_argument("--remote", default="origin", help="ssh remote to parse") - parser.add_argument( - "--pr-body", help="(testing) PR body to use instead of fetching from GitHub" - ) - args = parser.parse_args() - - branch = git(["rev-parse", "--abbrev-ref", "HEAD"]) - - # Don't skip slow tests on main or ci-docker-staging - skip_branches = {"main", "ci-docker-staging"} - if branch in skip_branches: - print(f"Branch {branch} is in {skip_branches}, running slow tests") - exit(1) - print(f"Branch {branch} is not in {skip_branches}, checking last commit...") - - if args.pr_body: - body = args.pr_body - else: - remote = git(["config", "--get", f"remote.{args.remote}.url"]) - user, repo = parse_remote(remote) - - github = GitHubRepo(token=os.environ["GITHUB_TOKEN"], user=user, repo=repo) - pr = github.get(f"pulls/{args.pr}") - body = pr["body"] - - body_match, reason = check_match(body, SLOW_TEST_TRIGGERS) - - if body_match: - print(f"Matched {reason} in PR body:\n{display(body)}, running slow tests") - exit(1) - - print( - f"PR Body:\n{display(body)}\ndid not have any of {SLOW_TEST_TRIGGERS}, skipping slow tests" - ) - exit(0) diff --git a/ci/scripts/package/README.md b/ci/scripts/package/README.md deleted file mode 100644 index 972c315309..0000000000 --- a/ci/scripts/package/README.md +++ /dev/null @@ -1,31 +0,0 @@ -<!--- Licensed to the Apache Software Foundation (ASF) under one --> -<!--- or more contributor license agreements. See the NOTICE file --> -<!--- distributed with this work for additional information --> -<!--- regarding copyright ownership. The ASF licenses this file --> -<!--- to you under the Apache License, Version 2.0 (the --> -<!--- "License"); you may not use this file except in compliance --> -<!--- with the License. You may obtain a copy of the License at --> - -<!--- http://www.apache.org/licenses/LICENSE-2.0 --> - -<!--- Unless required by applicable law or agreed to in writing, --> -<!--- software distributed under the License is distributed on an --> -<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --> -<!--- KIND, either express or implied. See the License for the --> -<!--- specific language governing permissions and limitations --> -<!--- under the License. --> - -# TVM wheel packaging - -The wheels are built by a standard `cibuildwheel` flow, configured in -`.github/workflows/publish_wheel.yml` and `pyproject.toml` (`[tool.cibuildwheel]` -and `[tool.scikit-build]`). This directory holds the few helper scripts that flow -invokes: - -- `manylinux_build_libtvm_runtime_cuda.sh` — run by the `build_cuda_runtime` CI - stage; builds the `libtvm_runtime_cuda.so` sidecar inside the prebuilt - `quay.io/manylinux_cuda` image (CUDA toolkit preinstalled). -- `windows_build_libtvm_runtime_cuda.bat` — the Windows equivalent (run with - `shell: cmd`); installs the CUDA toolkit via conda and builds - `tvm_runtime_cuda.dll`. -- `build-environment.yaml` — conda environment for building the wheel. diff --git a/ci/scripts/package/build-environment.yaml b/ci/scripts/package/build-environment.yaml deleted file mode 100644 index 3b2c4dd167..0000000000 --- a/ci/scripts/package/build-environment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -# Build environment for TVM wheel building. -# This environment provides the necessary dependencies for building TVM wheels. -name: tvm-wheel-build - -# The conda channels to lookup the dependencies -channels: - - conda-forge - -# The packages to install to the environment -dependencies: - # Core build tools - - cmake >=3.24 - - ninja - - make - - llvmdev >=11 - - python >=3.10 - - pip - - git - - bzip2 - - zlib - - zstd-static - - pytest - - numpy - - scipy - - cython - - libxml2-devel diff --git a/ci/scripts/package/manylinux_build_libtvm_runtime_cuda.sh b/ci/scripts/package/manylinux_build_libtvm_runtime_cuda.sh deleted file mode 100755 index d0d847ae83..0000000000 --- a/ci/scripts/package/manylinux_build_libtvm_runtime_cuda.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Build libtvm_runtime_cuda.so inside a manylinux CUDA container, run by the -# build_cuda_runtime CI job. The official quay.io/manylinux_cuda images ship -# the CUDA toolkit preinstalled under /usr/local/cuda, but omit the libcuda -# driver stub, so we install just cuda-driver-devel from the image's CUDA repo. -# Builds the sidecar into build-wheel-cuda/lib/ for the wheel build to bundle. -# -# Usage: manylinux_build_libtvm_runtime_cuda.sh -set -euxo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -build_dir="${repo_root}/build-wheel-cuda" -parallel="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" - -# The image ships the toolkit but not the libcuda driver stub; install it from -# the image's CUDA repo so the sidecar links libcuda.so.1 (sync version with tag). -dnf -y install cuda-driver-devel-13-1 - -# Build the CUDA runtime sidecar with CUDA on and LLVM off, so it does not need -# the LLVM prefix; the main CPU wheel links LLVM statically. The manylinux CUDA -# image already ships cmake and make, and the build uses the default Makefiles -# generator (no Ninja), so no build tools are installed here. Put the bundled -# CPython and CUDA toolchain on PATH for the CMake configure and nvcc. -export PATH="/opt/python/cp310-cp310/bin:/usr/local/cuda/bin:${PATH}" -nvcc --version - -rm -rf "${build_dir}" -# CMAKE_CUDA_COMPILER only tells CMake which nvcc to use; it does not affect the -# resulting libtvm_runtime_cuda.so, which is built only from .cc host sources (no -# .cu device code, so nvcc is never invoked for it). CMAKE_CUDA_ARCHITECTURES is -# intentionally not set: it would be a no-op here for the same reason (verified -- -# the .so is byte-identical across arch values and carries no device code), and -# modern CMake fills in a default so configure does not fail without it. -cmake -S "${repo_root}" -B "${build_dir}" \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=OFF \ - -DTVM_BUILD_PYTHON_MODULE=ON \ - -DUSE_CUDA=/usr/local/cuda \ - -DUSE_LLVM=OFF \ - -DUSE_CUBLAS=OFF -DUSE_CUDNN=OFF -DUSE_CUTLASS=OFF -DUSE_NCCL=OFF -DUSE_NVTX=OFF \ - -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -cmake --build "${build_dir}" --target tvm_runtime tvm_runtime_cuda --parallel "${parallel}" - -cuda_lib="${build_dir}/lib/libtvm_runtime_cuda.so" -test -f "${cuda_lib}" -patchelf --set-rpath '$ORIGIN' "${cuda_lib}" -echo "CUDA runtime: ${cuda_lib}" diff --git a/ci/scripts/package/windows_build_libtvm_runtime_cuda.bat b/ci/scripts/package/windows_build_libtvm_runtime_cuda.bat deleted file mode 100644 index 4864c18a80..0000000000 --- a/ci/scripts/package/windows_build_libtvm_runtime_cuda.bat +++ /dev/null @@ -1,99 +0,0 @@ -@echo off -rem Licensed to the Apache Software Foundation (ASF) under one -rem or more contributor license agreements. See the NOTICE file -rem distributed with this work for additional information -rem regarding copyright ownership. The ASF licenses this file -rem to you under the Apache License, Version 2.0 (the -rem "License"); you may not use this file except in compliance -rem with the License. You may obtain a copy of the License at -rem -rem http://www.apache.org/licenses/LICENSE-2.0 -rem -rem Unless required by applicable law or agreed to in writing, -rem software distributed under the License is distributed on an -rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -rem KIND, either express or implied. See the License for the -rem specific language governing permissions and limitations -rem under the License. -rem -rem Build tvm_runtime_cuda.dll on a Windows runner, run by the build_cuda_runtime -rem CI job (on the host; unlike Linux there is no build container on Windows). -rem Installs the pinned CUDA toolkit via conda and builds the sidecar into -rem build-wheel-cuda\lib\ for the wheel build to bundle. Windows mirror of -rem manylinux_build_libtvm_runtime_cuda.sh. Run with: shell: cmd. -setlocal enableextensions - -rem repo root = this script's directory / ..\..\.. (native Windows paths; no cygpath). -pushd "%~dp0..\..\.." || exit /b 1 -set "repo_root=%CD%" -popd -set "build_dir=%repo_root%\build-wheel-cuda" -set "cuda_prefix=C:\opt\cuda" - -rem Locate conda: the runner ships Miniconda (exposed via %CONDA%) but it may not be -rem on PATH in this shell. -set "conda_exe=conda" -where conda >nul 2>nul || set "conda_exe=%CONDA%\Scripts\conda.exe" - -rem Install the pinned CUDA toolkit via conda from the nvidia channel, mirroring the -rem LLVM-via-conda install used elsewhere. The win-64 channel caps at 13.0.x, so this -rem pins 13.0.2 -- slightly behind the Linux image's CUDA 13.1, which is harmless: the -rem sidecar has no device code and links the CUDA runtime by soname only. The nvidia -rem CDN occasionally returns a transient HTTP 5xx, so retry once; a half-finished first -rem attempt can leave the prefix partially populated, so wipe it before retrying. -if not exist "%cuda_prefix%\Library\bin\nvcc.exe" ( - call "%conda_exe%" create -q -p "%cuda_prefix%" -c nvidia/label/cuda-13.0.2 cuda-toolkit -y - if errorlevel 1 ( - if exist "%cuda_prefix%" rmdir /s /q "%cuda_prefix%" - call "%conda_exe%" create -q -p "%cuda_prefix%" -c nvidia/label/cuda-13.0.2 cuda-toolkit -y || exit /b 1 - ) -) - -rem conda lays the Windows toolkit out under <prefix>\Library (bin\nvcc.exe, -rem lib\x64\cudart.lib, include\...). Discover the root from nvcc.exe so TVM's -rem FindCUDA MSVC branch resolves against the real layout instead of a hardcode. -set "nvcc_exe=" -for /f "delims=" %%i in ('dir /s /b "%cuda_prefix%\nvcc.exe" 2^>nul') do if not defined nvcc_exe set "nvcc_exe=%%i" -if not defined nvcc_exe ( echo nvcc.exe not found under %cuda_prefix% & exit /b 1 ) -rem cuda_root = dirname(dirname(nvcc)) = <prefix>\Library -for %%i in ("%nvcc_exe%") do set "nvcc_bin=%%~dpi" -pushd "%nvcc_bin%.." || exit /b 1 -set "cuda_root=%CD%" -popd -set "CUDA_PATH=%cuda_root%" - -python -m pip install -U pip cmake ninja || exit /b 1 -"%nvcc_exe%" --version || exit /b 1 - -rem nvcc needs the MSVC host compiler (cl.exe), so locate VS via vswhere and run the -rem cmake configure+build inside vcvars64 (this shell is not a VS Developer prompt). -set "vswhere=C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -set "vs_path=" -for /f "usebackq delims=" %%i in (`"%vswhere%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "vs_path=%%i" -if not defined vs_path ( echo Visual Studio with VC tools not found & exit /b 1 ) -set "vcvars=%vs_path%\VC\Auxiliary\Build\vcvars64.bat" - -if exist "%build_dir%" rmdir /s /q "%build_dir%" - -rem CMAKE_CUDA_COMPILER only tells CMake which nvcc to use (load-bearing: the conda -rem nvcc is not on PATH); it does not affect the resulting tvm_runtime_cuda.dll, which -rem is built only from .cc host sources (no .cu device code). CMAKE_CUDA_ARCHITECTURES -rem is intentionally not set -- a no-op for the same reason, and modern CMake fills a -rem default. -allow-unsupported-compiler guards against the runner's MSVC being newer -rem than the CUDA toolkit officially supports. The cmake command is kept on one line: -rem `^` continuations in a batch file break on any trailing whitespace. -rem CMake parses backslashes in string values as escapes (e.g. C:\opt -> invalid \o), -rem so hand cmake forward-slash paths. cmd builtins (rmdir / if exist) keep backslashes. -set "repo_root_fwd=%repo_root:\=/%" -set "build_dir_fwd=%build_dir:\=/%" -set "cuda_root_fwd=%cuda_root:\=/%" -set "nvcc_fwd=%nvcc_exe:\=/%" - -call "%vcvars%" || exit /b 1 -cmake -S "%repo_root_fwd%" -B "%build_dir_fwd%" -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DTVM_BUILD_PYTHON_MODULE=ON -DUSE_CUDA="%cuda_root_fwd%" -DUSE_LLVM=OFF -DUSE_CUBLAS=OFF -DUSE_CUDNN=OFF -DUSE_CUTLASS=OFF -DUSE_NCCL=OFF -DUSE_NVTX=OFF -DCMAKE_CUDA_COMPILER="%nvcc_fwd%" -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler" || exit /b 1 -cmake --build "%build_dir_fwd%" --target tvm_runtime tvm_runtime_cuda --config Release || exit /b 1 - -if not exist "%build_dir%\lib\tvm_runtime_cuda.dll" ( echo tvm_runtime_cuda.dll was not produced & exit /b 1 ) -rem No patchelf/rpath step on Windows; delvewheel vendors dependencies at repair time. -echo CUDA runtime: %build_dir%\lib\tvm_runtime_cuda.dll -endlocal diff --git a/tests/python/ci/jenkins_loader_test.groovy b/tests/python/ci/jenkins_loader_test.groovy new file mode 100644 index 0000000000..28d0df3d74 --- /dev/null +++ b/tests/python/ci/jenkins_loader_test.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.control.CompilerConfiguration + +class LoaderPipeline extends Script { + Object run() {} + Object evaluate(String source) { + state.evaluated << source + if (state.evaluationFailure) throw state.evaluationFailure + return 'loaded helper' + } + def readTrusted(String path) { + assert path == 'ci/jenkins/generated/common.groovy' + if (state.readFailure) throw state.readFailure + return 'selected revision helper' + } + def node(String label, Closure body) { + assert label == 'CPU-SMALL' + state.nodes++ + body() + } + def sh(Map options) { + state.fetches++ + assert options.returnStdout + assert options.script.contains('https://github.com/apache/tvm.git') + assert options.script.contains('+refs/heads/main:refs/heads/trusted-main') + assert !options.script.contains('CHANGE_TARGET') + if (state.fetchFailure) throw state.fetchFailure + return 'latest main helper' + } +} + +def loader = new GroovyClassLoader(getClass().classLoader) +def abort = loader.parseClass('package hudson; class AbortException extends IOException { AbortException(String message) { super(message) } }') +def config = new CompilerConfiguration(scriptBaseClass: 'LoaderPipeline') +def helper = 'ci/jenkins/generated/common.groovy' +def modified = { abort.newInstance("‘${helper}’ has been modified in an untrusted revision".toString()) } +def missing = { new FileNotFoundException(helper) } +def cases = [ + [name: 'unchanged PR'], + [name: 'trusted named branch', branch: true], + [name: 'modified PR', readFailure: modified(), fallback: true], + [name: 'deleted PR lightweight', readFailure: missing(), fallback: true], + [name: 'deleted PR heavyweight', readFailure: abort.newInstance("/workspace/trusted/${helper} not found".toString()), fallback: true], + [name: 'release target PR', readFailure: modified(), fallback: true, target: 'release'], + [name: 'missing named branch', branch: true, readFailure: missing()], + [name: 'rejected named branch', branch: true, readFailure: modified()], + [name: 'unrelated abort', readFailure: abort.newInstance('controller offline')], + [name: 'unrelated missing file', readFailure: new FileNotFoundException('other/file')], + [name: 'read network failure', readFailure: new IOException('network')], + [name: 'interrupted read', readFailure: new InterruptedException('cancelled')], + [name: 'fetch failure', readFailure: modified(), fallback: true, fetchFailure: new IOException('fetch failed')], + [name: 'missing main helper', readFailure: missing(), fallback: true, fetchFailure: abort.newInstance('main helper missing')], + [name: 'selected evaluation error', evaluationFailure: new IllegalStateException('bad helper')], + [name: 'fallback evaluation error', readFailure: modified(), fallback: true, evaluationFailure: new IllegalStateException('bad main helper')], +] +int count = 0 +args.each { filename -> + def fullSource = new File(filename).text + def source = fullSource.substring(0, fullSource.indexOf('\nci.start()')) + cases.each { scenario -> + def state = [nodes: 0, fetches: 0, evaluated: []] + scenario + def binding = new Binding([state: state, env: [CHANGE_ID: scenario.branch ? null : '123', CHANGE_TARGET: scenario.target ?: 'main']]) + Throwable failure + try { + new GroovyShell(loader, binding, config).evaluate(source) + } catch (Throwable caught) { + failure = caught + } + def expected = scenario.fetchFailure ?: scenario.evaluationFailure ?: (scenario.fallback ? null : scenario.readFailure) + assert failure.is(expected): "${filename}: ${scenario.name}: ${failure} != ${expected}" + assert state.nodes == (scenario.fallback ? 1 : 0): scenario.name + assert state.fetches == (scenario.fallback ? 1 : 0): scenario.name + def shouldEvaluate = (!scenario.readFailure || scenario.fallback) && !scenario.fetchFailure + assert state.evaluated == (shouldEvaluate ? [scenario.fallback ? 'latest main helper' : 'selected revision helper'] : []): scenario.name + count++ + } +} +println("${count} native loader cases passed") diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py index 5e71cc7c0f..7144fee3d1 100644 --- a/tests/python/ci/test_ci.py +++ b/tests/python/ci/test_ci.py @@ -23,15 +23,12 @@ from pathlib import Path import pytest -from .test_utils import JENKINS_SCRIPT_ROOT, REPO_ROOT, TempGit, run_script +from .test_utils import JENKINS_SCRIPT_ROOT, TempGit, run_script [email protected]( - params=[JENKINS_SCRIPT_ROOT, REPO_ROOT / "ci/scripts/jenkins"], ids=["native", "legacy"] -) -def jenkins_script_root(request): - # Fork PR jobs still use main's Jenkinsfiles and the legacy command paths. - return request.param [email protected] +def jenkins_script_root(): + return JENKINS_SCRIPT_ROOT def parameterize_named(**kwargs): @@ -314,49 +311,3 @@ def test_should_rebuild_docker( if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) - - [email protected]( - "files, expected", - [([], 0), (["docs/page.rst"], 1), (["src/code.cc"], 0), (["docs/page.rst", "src/code.cc"], 0)], -) -def test_legacy_docs_gate(tmp_path, files, expected): - repo = TempGit(tmp_path) - repo.run("config", "user.name", "CI Test") - repo.run("config", "user.email", "[email protected]") - repo.run("commit", "--allow-empty", "-m", "base") - repo.run("update-ref", "refs/remotes/origin/main", "HEAD") - for filename in files: - path = tmp_path / filename - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("change") - repo.run("add", ".") - repo.run("commit", "--allow-empty", "-m", "change") - result = run_script( - [REPO_ROOT / "ci/scripts/jenkins/git_change_docs.sh"], cwd=tmp_path, check=False - ) - assert result.returncode == expected - - [email protected]( - "branch, body, expected", - [ - ("main", "ordinary change", 1), - ("ci-docker-staging", "ordinary change", 1), - ("feature", "ordinary change", 0), - ("feature", "@tvm-bot run slow tests", 1), - ], -) -def test_legacy_slow_gate(tmp_path, branch, body, expected): - repo = TempGit(tmp_path) - repo.run("config", "user.name", "CI Test") - repo.run("config", "user.email", "[email protected]") - repo.run("commit", "--allow-empty", "-m", "base") - if branch != "main": - repo.run("checkout", "-b", branch) - result = run_script( - [REPO_ROOT / "ci/scripts/jenkins/should_run_slow_tests.py", "--pr", "1", "--pr-body", body], - cwd=tmp_path, - check=False, - ) - assert result.returncode == expected diff --git a/tests/python/ci/test_jenkins_helpers.py b/tests/python/ci/test_jenkins_helpers.py index cfc0c69ced..f521d41303 100644 --- a/tests/python/ci/test_jenkins_helpers.py +++ b/tests/python/ci/test_jenkins_helpers.py @@ -40,11 +40,9 @@ def executable(path, body): path.chmod(0o755) [email protected]( - "script_root", [JENKINS_SCRIPT_ROOT, REPO_ROOT / "ci/scripts/jenkins"], ids=["native", "legacy"] -) @pytest.mark.parametrize("failures", [2, 3]) -def test_s3_transfer_retries(tmp_path, monkeypatch, failures, script_root): +def test_s3_transfer_retries(tmp_path, monkeypatch, failures): + script_root = JENKINS_SCRIPT_ROOT # Run the real retry.sh with fake AWS and sleep, without network or delays. executable( tmp_path / "aws", @@ -70,7 +68,7 @@ with pathlib.Path(os.environ["SLEEP_LOG"]).open("a") as out: monkeypatch.setenv("FAILURES", str(failures)) monkeypatch.setenv("SLEEP_LOG", str(tmp_path / "sleeps")) monkeypatch.syspath_prepend(str(script_root)) - destination = "file name" if script_root == JENKINS_SCRIPT_ROOT else "artifact.so" + destination = "file name" for transfer in range(2): log = tmp_path / f"aws-{transfer}" monkeypatch.setenv("AWS_LOG", str(log)) diff --git a/tests/python/ci/test_jenkins_loader.py b/tests/python/ci/test_jenkins_loader.py new file mode 100644 index 0000000000..2907c94b78 --- /dev/null +++ b/tests/python/ci/test_jenkins_loader.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Exercise the native helper trust boundary and its isolated Git fetch.""" + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from .test_utils import REPO_ROOT, TempGit + +PIPELINES = sorted((REPO_ROOT / "ci/jenkins/generated").glob("*_jenkinsfile.groovy")) +HELPER = "ci/jenkins/generated/common.groovy" + + +def test_native_loader_trust_boundary(): + groovy = shutil.which("groovy") + if groovy is None: + pytest.skip("Groovy is required for the executable pipeline loader checks") + subprocess.run( + [groovy, str(Path(__file__).with_name("jenkins_loader_test.groovy")), *PIPELINES], + check=True, + ) + + [email protected]("failure", [None, "fetch", "missing-helper"]) +def test_trusted_main_fetch(tmp_path, failure): + # Execute the actual loader shell against a local upstream. A Git shim only + # substitutes the transport URL; ref selection and object reads use real Git. + upstream = tmp_path / "upstream" + upstream.mkdir() + repo = TempGit(upstream) + repo.run("config", "user.name", "CI Test") + repo.run("config", "user.email", "[email protected]") + helper = upstream / HELPER + helper.parent.mkdir(parents=True) + helper.write_text("old main helper") + repo.run("add", ".") + repo.run("commit", "-m", "initial main") + repo.run("branch", "release") + helper.write_text("latest main helper") + repo.run("commit", "-am", "advance main") + if failure == "missing-helper": + helper.unlink() + repo.run("commit", "-am", "remove helper") + + checkout = tmp_path / "untrusted-checkout" + checkout.mkdir() + untrusted_helper = checkout / HELPER + untrusted_helper.parent.mkdir(parents=True) + untrusted_helper.write_text("rejected PR helper") + binary_dir = tmp_path / "bin" + binary_dir.mkdir() + git = binary_dir / "git" + git.write_text( + "#!/usr/bin/env python3\n" + "import os, sys\n" + "args = sys.argv[1:]\n" + "url = 'https://github.com/apache/tvm.git'\n" + "if 'fetch' in args:\n" + " assert url in args\n" + " assert '+refs/heads/main:refs/heads/trusted-main' in args\n" + " if os.environ.get('FAIL_FETCH'): sys.exit(7)\n" + " args[args.index(url)] = os.environ['TEST_UPSTREAM']\n" + "os.execv(os.environ['REAL_GIT'], ['git'] + args)\n" + ) + git.chmod(0o755) + env = dict( + os.environ, + PATH=f"{binary_dir}:{os.environ['PATH']}", + REAL_GIT=shutil.which("git"), + TEST_UPSTREAM=str(upstream), + FAIL_FETCH="yes" if failure == "fetch" else "", + CHANGE_TARGET="release", + ) + scripts = { + p.read_text().split("label: 'Read trusted helper from upstream main',")[1].split("'" * 3)[1] + for p in PIPELINES + } + assert len(scripts) == 1 + result = subprocess.run( + ["sh", "-c", scripts.pop()], cwd=checkout, env=env, text=True, capture_output=True + ) + if failure: + assert result.returncode != 0 + assert result.stdout == "" + else: + assert result.returncode == 0, result.stderr + assert result.stdout == "latest main helper" + assert untrusted_helper.read_text() == "rejected PR helper" diff --git a/tests/scripts/task_show_node_info.sh b/tests/scripts/task_show_node_info.sh deleted file mode 100755 index af095ed453..0000000000 --- a/tests/scripts/task_show_node_info.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euxo pipefail - -echo "===== JENKINS INFO =====" -echo "NODE_NAME=$NODE_NAME" -echo "EXECUTOR_NUMBER=$EXECUTOR_NUMBER" -echo "WORKSPACE=$WORKSPACE" -echo "BUILD_NUMBER=$BUILD_NUMBER" -echo "WORKSPACE=$WORKSPACE" - -echo "===== EC2 INFO =====" -function ec2_metadata() { - # See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html - curl -w '\n' -fsSL "http://169.254.169.254/latest/meta-data/$1"; || echo failed -} - -ec2_metadata ami-id -ec2_metadata instance-id -ec2_metadata instance-type -ec2_metadata hostname -ec2_metadata public-hostname - -echo "===== RUNNER INFO =====" -df --human-readable -lscpu -free -nvidia-smi 2>/dev/null || echo "CUDA not found"
