This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-cleanup-trusted-main in repository https://gitbox.apache.org/repos/asf/tvm.git
commit d5089384b5410ba2ba1a97dfc0bc635699874fef Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 02:28:30 2026 +0000 [REFACTOR][CI] Centralize artifact selection behind compatible CLI options --- ci/jenkins/README.md | 44 +++++-- ci/jenkins/{data.py => artifacts.py} | 13 +- ci/jenkins/generated/arm_jenkinsfile.groovy | 41 +----- ci/jenkins/generated/common.groovy | 2 +- ci/jenkins/generated/cpu_jenkinsfile.groovy | 41 +----- ci/jenkins/generated/docker_jenkinsfile.groovy | 39 +----- ci/jenkins/generated/gpu_jenkinsfile.groovy | 45 +------ ci/jenkins/generated/wasm_jenkinsfile.groovy | 39 +----- ci/jenkins/scripts/s3.py | 83 ++++++++---- tests/python/ci/jenkins_loader_test.groovy | 34 ++--- tests/python/ci/test_jenkins_helpers.py | 175 +++++++++++++++++++++++++ tests/python/ci/test_jenkins_loader.py | 73 +---------- 12 files changed, 300 insertions(+), 329 deletions(-) diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 495ff64cb8..e33f38da4c 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -45,7 +45,23 @@ Despite the historical directory name, these are native Groovy sources: edit and review them directly. `generated/common.groovy` shares preparation, checkout, Docker setup, artifacts and spot fallback policy. Platform commands and deployment stay visible in their respective job files. Image tags remain in -`docker-images.ini`; artifact bundles remain in `data.py`. +`docker-images.ini`; artifact bundles remain in `artifacts.py`. Bundle entries may +contain repository-relative globs such as `build/dist/*.whl`. Upload expands +patterns locally, sorts and deduplicates the paths, and keeps their relative +paths under the S3 prefix. Unmatched patterns produce a warning; missing literal +optional artifacts retain their warning-and-skip behavior. Directories keep +recursive upload behavior. Download still copies the complete prefix without +expanding local or remote globs. +Use `s3.py --artifacts NAME...` to select registry entries or `--paths PATH...` +to select raw paths; the selections are mutually exclusive. Existing Jenkinsfiles +can keep `--bundle` and `--items`, which are aliases with the same semantics. +The shared helper's legacy `upload(tag, options)` passes those options unchanged. +`artifacts.py` exposes the declarative `ARTIFACTS` registry; there is no `data.py` +compatibility module. + +The shared helper groups internal dependencies first, public entry points next, +and retained legacy functions last. Existing function signatures, shared variables, +startup and retry behavior remain available to the platform Jenkinsfiles. Host-side commands live in `scripts/`. GitHub bot and release API helpers live in `../github/`. The retained `determine_docker_images.py` supports both @@ -54,24 +70,24 @@ and the standalone skip commands retain their command-line interfaces. ### Trust and rollout -Each entry point first reads the shared helper with `readTrusted` at Jenkins' -selected SCM revision. Trusted named branches retain their own helper. For PRs, -Jenkins rejects an untrusted helper modification and may report a deleted helper -as missing. These specific read failures recover by fetching `refs/heads/main` -from the fixed `https://github.com/apache/tvm.git` URL into a fresh temporary bare -repository on a small CPU node. The fetched commit is resolved and its helper -blob is read directly, without checking out or evaluating PR-owned code. Recovery -always uses latest upstream main, including for release-target PRs. - -Other read failures, cancellation, fallback fetch/read failures and helper -evaluation errors propagate. Evaluation occurs outside the read-recovery block. -The fallback requires no credentials or controller configuration changes. +Each entry point evaluates the shared helper returned by `readTrusted` at +Jenkins' selected SCM revision. Untrusted PRs that modify or delete the helper +fail immediately, before node allocation, checkout, or credentials. Missing +helpers, interrupted reads, and evaluation errors also propagate directly. +No alternative helper is fetched or evaluated. Trusted named branches retain +their selected helper revision. Cancellation runs once before preparation. During preparation, the normal source merge pins the PR target revision for subsequent stages, including release targets. Host scripts and artifact metadata follow that pinned target revision for PR authors outside `CONTRIBUTORS.md`. +The pinned target must contain the coordinated `artifacts.py` and `s3.py` update. +Before that update reaches a target, a Jenkins-trusted PR from an author outside +its contributor list cannot restore the renamed registry, and that build fails. +Use a trusted named branch for the transition; do not bypass the trusted-file +checkout or mix the old target's script with a PR-owned registry. + The obsolete `ci/scripts/` copies and old node-info path have been removed now that main's Jenkinsfiles use the canonical layout. The live `task_build.py`, source-mode test scripts and artifact interfaces remain available. Older target @@ -82,7 +98,7 @@ Before enabling the migration on production jobs, validate it with the existing Jenkins plugin versions and sandbox settings on a trusted branch: run all five entry points, suspend/resume across a node step, and exercise spot loss, a normal build-command failure, superseded builds, and a failed on-demand retry. Check -trusted helper changes, untrusted PRs without helper changes, and recovery of +trusted helper changes, untrusted PRs without helper changes, and refusal of untrusted modified/deleted helpers, including a release-target PR. Verify image overrides and disabled deployment gates before checking enabled docs/Docker deployment in the normal trusted deployment context. diff --git a/ci/jenkins/data.py b/ci/jenkins/artifacts.py similarity index 85% rename from ci/jenkins/data.py rename to ci/jenkins/artifacts.py index fd75107e7d..d777ba7fb7 100644 --- a/ci/jenkins/data.py +++ b/ci/jenkins/artifacts.py @@ -22,15 +22,18 @@ Single source of truth for the file lists uploaded to / downloaded from S3 by docker image tags live in ``ci/jenkins/docker-images.ini`` and pipeline configuration lives in ``ci/jenkins/generated/common.groovy``. -CLI: ``python3 ci/jenkins/data.py <bundle> [<bundle> ...]`` resolves bundle +CLI: ``python3 ci/jenkins/artifacts.py <bundle> [<bundle> ...]`` resolves bundle names to their file paths (one per line; exit 1 on unknown name). Used by ``s3.py`` at Jenkins runtime and by any external caller that needs -data-driven artifact lists. +data-driven artifact lists. Entries may be repository-relative literal paths or +glob patterns. Only s3.py upload expands patterns; this registry and its CLI +return the declarative entries unchanged. Downloads normally copy the whole +artifact prefix. """ import sys -files_to_stash = { +ARTIFACTS = { # Executables and build files needed to run c++ tests "cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"], # runtime files @@ -59,9 +62,9 @@ files_to_stash = { if __name__ == "__main__": paths = [] for name in sys.argv[1:]: - if name not in files_to_stash: + if name not in ARTIFACTS: print(f"unknown bundle: {name}", file=sys.stderr) sys.exit(1) - paths.extend(files_to_stash[name]) + paths.extend(ARTIFACTS[name]) for p in paths: print(p) diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index e34ad8fdf2..2ac6165a7e 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -15,43 +15,8 @@ // specific language governing permissions and limitations // under the License. -// Keep trusted branches on their selected helper; recover rejected PR helpers -// only from the fixed upstream main ref. Never evaluate the checkout's copy. -def helperPath = 'ci/jenkins/generated/common.groovy' -def helperSource -try { - helperSource = readTrusted(helperPath) -} catch (hudson.AbortException failure) { - def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" - def missing = failure.message?.endsWith("/${helperPath} not found") - if (!env.CHANGE_ID || !(rejected || missing)) { - throw failure - } -} catch (java.io.FileNotFoundException failure) { - if (!env.CHANGE_ID || failure.message != helperPath) { - throw failure - } -} -if (helperSource == null) { - node('CPU-SMALL') { - helperSource = sh( - returnStdout: true, - label: 'Read trusted helper from upstream main', - script: ''' - set -eu - helper_repo=$(mktemp -d) - trap 'rm -rf "$helper_repo"' EXIT - git init --bare --quiet "$helper_repo" - git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ - https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main - revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') - git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" - ''' - ) - } -} -// Evaluation and subsequent pipeline failures must never trigger a fallback. -ci = evaluate(helperSource) +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) ci.start() ci.build('ARM-GRAVITON3', 'BUILD: arm', 'tvm/build-arm', images.ci_arm, 'arm') { @@ -61,5 +26,5 @@ ci.build('ARM-GRAVITON3', 'BUILD: arm', 'tvm/build-arm', images.ci_arm, 'arm') { ) ci.cmake_build(images.ci_arm, 'build') ci.make_cpp_tests(images.ci_arm, 'build') - ci.upload('arm', '--bundle tvm_lib --bundle cpptest') + ci.upload('arm', '--artifacts tvm_lib --artifacts cpptest') } diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy index d4add87448..1682ca1147 100644 --- a/ci/jenkins/generated/common.groovy +++ b/ci/jenkins/generated/common.groovy @@ -82,7 +82,7 @@ def checkout_trusted_files() { // (especially those that access secrets) should be checked out here so // only trusted versions are used in CI sh( - script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/. ci/jenkins/data.py", + script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/. ci/jenkins/artifacts.py", label: 'Check out trusted files', ) } diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 119fd242a6..881cd2cedc 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -15,43 +15,8 @@ // specific language governing permissions and limitations // under the License. -// Keep trusted branches on their selected helper; recover rejected PR helpers -// only from the fixed upstream main ref. Never evaluate the checkout's copy. -def helperPath = 'ci/jenkins/generated/common.groovy' -def helperSource -try { - helperSource = readTrusted(helperPath) -} catch (hudson.AbortException failure) { - def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" - def missing = failure.message?.endsWith("/${helperPath} not found") - if (!env.CHANGE_ID || !(rejected || missing)) { - throw failure - } -} catch (java.io.FileNotFoundException failure) { - if (!env.CHANGE_ID || failure.message != helperPath) { - throw failure - } -} -if (helperSource == null) { - node('CPU-SMALL') { - helperSource = sh( - returnStdout: true, - label: 'Read trusted helper from upstream main', - script: ''' - set -eu - helper_repo=$(mktemp -d) - trap 'rm -rf "$helper_repo"' EXIT - git init --bare --quiet "$helper_repo" - git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ - https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main - revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') - git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" - ''' - ) - } -} -// Evaluation and subsequent pipeline failures must never trigger a fallback. -ci = evaluate(helperSource) +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) ci.start() ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') { @@ -61,7 +26,7 @@ ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') { ) ci.cmake_build(images.ci_cpu, 'build') ci.make_cpp_tests(images.ci_cpu, 'build') - ci.upload('cpu', '--bundle tvm_lib --bundle cpptest') + ci.upload('cpu', '--artifacts tvm_lib --artifacts cpptest') } stage('Test') { diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index b4b67852ba..0039be7712 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -15,43 +15,8 @@ // specific language governing permissions and limitations // under the License. -// Keep trusted branches on their selected helper; recover rejected PR helpers -// only from the fixed upstream main ref. Never evaluate the checkout's copy. -def helperPath = 'ci/jenkins/generated/common.groovy' -def helperSource -try { - helperSource = readTrusted(helperPath) -} catch (hudson.AbortException failure) { - def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" - def missing = failure.message?.endsWith("/${helperPath} not found") - if (!env.CHANGE_ID || !(rejected || missing)) { - throw failure - } -} catch (java.io.FileNotFoundException failure) { - if (!env.CHANGE_ID || failure.message != helperPath) { - throw failure - } -} -if (helperSource == null) { - node('CPU-SMALL') { - helperSource = sh( - returnStdout: true, - label: 'Read trusted helper from upstream main', - script: ''' - set -eu - helper_repo=$(mktemp -d) - trap 'rm -rf "$helper_repo"' EXIT - git init --bare --quiet "$helper_repo" - git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ - https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main - revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') - git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" - ''' - ) - } -} -// Evaluation and subsequent pipeline failures must never trigger a fallback. -ci = evaluate(helperSource) +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) ci.start() def ecr_push(full_name) { diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 47ccb0b5c0..98c04b332e 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -15,49 +15,14 @@ // specific language governing permissions and limitations // under the License. -// Keep trusted branches on their selected helper; recover rejected PR helpers -// only from the fixed upstream main ref. Never evaluate the checkout's copy. -def helperPath = 'ci/jenkins/generated/common.groovy' -def helperSource -try { - helperSource = readTrusted(helperPath) -} catch (hudson.AbortException failure) { - def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" - def missing = failure.message?.endsWith("/${helperPath} not found") - if (!env.CHANGE_ID || !(rejected || missing)) { - throw failure - } -} catch (java.io.FileNotFoundException failure) { - if (!env.CHANGE_ID || failure.message != helperPath) { - throw failure - } -} -if (helperSource == null) { - node('CPU-SMALL') { - helperSource = sh( - returnStdout: true, - label: 'Read trusted helper from upstream main', - script: ''' - set -eu - helper_repo=$(mktemp -d) - trap 'rm -rf "$helper_repo"' EXIT - git init --bare --quiet "$helper_repo" - git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ - https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main - revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') - git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" - ''' - ) - } -} -// Evaluation and subsequent pipeline failures must never trigger a fallback. -ci = evaluate(helperSource) +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) ci.start() ci.build('CPU', 'BUILD: GPU', 'tvm/build-gpu', images.ci_gpu, 'gpu') { sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" ci.cmake_build("${images.ci_gpu} --no-gpu", 'build') - ci.upload('gpu', '--bundle tvm_lib --bundle tvm_lib_gpu_extra') + ci.upload('gpu', '--artifacts tvm_lib --artifacts tvm_lib_gpu_extra') // Check the compiler-only GPU configuration after publishing the test build. sh 'rm -rf build' @@ -87,7 +52,7 @@ stage('Test') { script: "${docker_run} ${images.ci_gpu} ./tests/scripts/task_python_docs.sh", label: 'Build docs', ) - ci.upload('docs', '--items docs.tgz') + ci.upload('docs', '--paths docs.tgz') sh( script: "aws s3 cp --no-progress _docs s3://${s3_bucket}/${s3_prefix}/docs --recursive", label: 'Upload docs to S3', @@ -143,7 +108,7 @@ stage('Deploy') { ci.deploy_step('Deploy Docs', env.DOCS_DEPLOY_ENABLED == 'yes', 'tvm/deploy-docs') { ci.init_git() sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/docs --items docs.tgz", + script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/docs --paths docs.tgz", label: 'Download docs folder from S3', ) deploy_docs() diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index ac4b075721..ddf932529d 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -15,43 +15,8 @@ // specific language governing permissions and limitations // under the License. -// Keep trusted branches on their selected helper; recover rejected PR helpers -// only from the fixed upstream main ref. Never evaluate the checkout's copy. -def helperPath = 'ci/jenkins/generated/common.groovy' -def helperSource -try { - helperSource = readTrusted(helperPath) -} catch (hudson.AbortException failure) { - def rejected = failure.message == "‘${helperPath}’ has been modified in an untrusted revision" - def missing = failure.message?.endsWith("/${helperPath} not found") - if (!env.CHANGE_ID || !(rejected || missing)) { - throw failure - } -} catch (java.io.FileNotFoundException failure) { - if (!env.CHANGE_ID || failure.message != helperPath) { - throw failure - } -} -if (helperSource == null) { - node('CPU-SMALL') { - helperSource = sh( - returnStdout: true, - label: 'Read trusted helper from upstream main', - script: ''' - set -eu - helper_repo=$(mktemp -d) - trap 'rm -rf "$helper_repo"' EXIT - git init --bare --quiet "$helper_repo" - git --git-dir="$helper_repo" fetch --quiet --no-tags --depth=1 \ - https://github.com/apache/tvm.git +refs/heads/main:refs/heads/trusted-main - revision=$(git --git-dir="$helper_repo" rev-parse --verify 'refs/heads/trusted-main^{commit}') - git --git-dir="$helper_repo" show "$revision:ci/jenkins/generated/common.groovy" - ''' - ) - } -} -// Evaluation and subsequent pipeline failures must never trigger a fallback. -ci = evaluate(helperSource) +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy')) ci.start() ci.build('CPU', 'BUILD: WASM', 'tvm/build-wasm', images.ci_wasm, 'wasm') { diff --git a/ci/jenkins/scripts/s3.py b/ci/jenkins/scripts/s3.py index c0859228d2..f48986cb58 100755 --- a/ci/jenkins/scripts/s3.py +++ b/ci/jenkins/scripts/s3.py @@ -17,6 +17,7 @@ # under the License. import argparse +import glob import importlib.util import logging import re @@ -26,30 +27,51 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[3] -DATA_PY = REPO_ROOT / "ci" / "jenkins" / "data.py" +ARTIFACTS_PY = REPO_ROOT / "ci" / "jenkins" / "artifacts.py" -def load_files_to_stash(): - """Load the files_to_stash dict from ci/jenkins/data.py.""" - spec = importlib.util.spec_from_file_location("data", DATA_PY) +def load_artifacts(): + """Load the ARTIFACTS dict from ci/jenkins/artifacts.py.""" + spec = importlib.util.spec_from_file_location("artifacts", ARTIFACTS_PY) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) - return mod.files_to_stash + return mod.ARTIFACTS -def resolve_bundles(bundle_names: list[str]) -> list[str]: - """Resolve a list of bundle names to a flat list of file paths.""" - files_to_stash = load_files_to_stash() +def resolve_artifacts(artifact_names: list[str]) -> list[str]: + """Resolve a list of artifact names to a flat list of file paths.""" + artifacts = load_artifacts() items = [] - for name in bundle_names: - if name not in files_to_stash: - known = list(files_to_stash.keys()) - logging.error(f"Unknown bundle '{name}'. Known bundles: {known}") + for name in artifact_names: + if name not in artifacts: + known = list(artifacts.keys()) + logging.error(f"Unknown artifact '{name}'. Known artifacts: {known}") raise SystemExit(1) - items.extend(files_to_stash[name]) + items.extend(artifacts[name]) return items +def resolve_upload_items(items: list[str]) -> list[str]: + """Expand local artifact patterns once, preserving literal path handling.""" + paths = set() + for item in items: + path = Path(item) + if glob.has_magic(item): + matches = glob.glob(item, root_dir=REPO_ROOT, recursive=True) + if not matches: + logging.warning("No artifacts matched pattern: %s", item) + paths.update(Path(match).as_posix() for match in matches) + else: + # Keep missing literal paths for the existing optional-artifact warning. + paths.add(path.as_posix()) + directories = {Path(path) for path in paths if Path(path).is_dir()} + # Recursive directory copies already include descendants selected by other + # entries or overlapping patterns. Transfer each destination only once. + return sorted( + path for path in paths if not any(parent in directories for parent in Path(path).parents) + ) + + RETRY_SCRIPT = REPO_ROOT / "ci" / "jenkins" / "scripts" / "retry.sh" S3_DOWNLOAD_REGEX = re.compile(r"download: s3://.* to (.*)") @@ -136,20 +158,26 @@ if __name__ == "__main__": parser.add_argument( "--prefix", help="s3 bucket + tag (e.g. s3://tvm-ci-prod/PR-1234/cpu", required=True ) - parser.add_argument("--items", help="files and folders to upload", nargs="+") - parser.add_argument( + selection = parser.add_mutually_exclusive_group() + selection.add_argument( + "--paths", + "--items", + dest="paths", + nargs="+", + help="repository-relative files, folders or upload glob patterns", + ) + selection.add_argument( + "--artifacts", "--bundle", - help="bundle name(s) from ci/jenkins/data.py files_to_stash (repeatable)", - action="append", - dest="bundles", + dest="artifacts", + nargs="+", + action="extend", + help="artifact names from ci/jenkins/artifacts.py (repeatable)", metavar="NAME", ) args = parser.parse_args() - if args.items is not None and args.bundles is not None: - parser.error("--items and --bundle are mutually exclusive") - logging.info(args) if Path.cwd() != REPO_ROOT: @@ -168,18 +196,21 @@ if __name__ == "__main__": logging.error(f"Unsupported action: {args.action}") exit(1) - if args.bundles is not None: - items = resolve_bundles(args.bundles) - elif args.items is not None: - items = args.items + if args.artifacts is not None: + items = resolve_artifacts(args.artifacts) + elif args.paths is not None: + items = args.paths else: if args.action == "upload": - logging.error("Cannot upload without --items or --bundle") + logging.error("Cannot upload without --paths or --artifacts") exit(1) else: # Download the whole prefix items = ["."] + if action == Action.UPLOAD: + items = resolve_upload_items(items) + for item in items: if action == Action.DOWNLOAD: source = s3_path diff --git a/tests/python/ci/jenkins_loader_test.groovy b/tests/python/ci/jenkins_loader_test.groovy index 28d0df3d74..1b2fe16e2e 100644 --- a/tests/python/ci/jenkins_loader_test.groovy +++ b/tests/python/ci/jenkins_loader_test.groovy @@ -30,19 +30,12 @@ class LoaderPipeline extends Script { return 'selected revision helper' } def node(String label, Closure body) { - assert label == 'CPU-SMALL' - state.nodes++ - body() + throw new AssertionError('loader allocated a node') } def sh(Map options) { - state.fetches++ - assert options.returnStdout - assert options.script.contains('https://github.com/apache/tvm.git') - assert options.script.contains('+refs/heads/main:refs/heads/trusted-main') - assert !options.script.contains('CHANGE_TARGET') - if (state.fetchFailure) throw state.fetchFailure - return 'latest main helper' + throw new AssertionError('loader ran a shell command') } + } def loader = new GroovyClassLoader(getClass().classLoader) @@ -54,20 +47,17 @@ def missing = { new FileNotFoundException(helper) } def cases = [ [name: 'unchanged PR'], [name: 'trusted named branch', branch: true], - [name: 'modified PR', readFailure: modified(), fallback: true], - [name: 'deleted PR lightweight', readFailure: missing(), fallback: true], - [name: 'deleted PR heavyweight', readFailure: abort.newInstance("/workspace/trusted/${helper} not found".toString()), fallback: true], - [name: 'release target PR', readFailure: modified(), fallback: true, target: 'release'], + [name: 'modified PR', readFailure: modified()], + [name: 'deleted PR lightweight', readFailure: missing()], + [name: 'deleted PR heavyweight', readFailure: abort.newInstance("/workspace/trusted/${helper} not found".toString())], + [name: 'release target PR', readFailure: modified(), target: 'release'], [name: 'missing named branch', branch: true, readFailure: missing()], [name: 'rejected named branch', branch: true, readFailure: modified()], [name: 'unrelated abort', readFailure: abort.newInstance('controller offline')], [name: 'unrelated missing file', readFailure: new FileNotFoundException('other/file')], [name: 'read network failure', readFailure: new IOException('network')], [name: 'interrupted read', readFailure: new InterruptedException('cancelled')], - [name: 'fetch failure', readFailure: modified(), fallback: true, fetchFailure: new IOException('fetch failed')], - [name: 'missing main helper', readFailure: missing(), fallback: true, fetchFailure: abort.newInstance('main helper missing')], [name: 'selected evaluation error', evaluationFailure: new IllegalStateException('bad helper')], - [name: 'fallback evaluation error', readFailure: modified(), fallback: true, evaluationFailure: new IllegalStateException('bad main helper')], ] int count = 0 args.each { filename -> @@ -82,12 +72,12 @@ args.each { filename -> } catch (Throwable caught) { failure = caught } - def expected = scenario.fetchFailure ?: scenario.evaluationFailure ?: (scenario.fallback ? null : scenario.readFailure) + def expected = scenario.readFailure ?: scenario.evaluationFailure assert failure.is(expected): "${filename}: ${scenario.name}: ${failure} != ${expected}" - assert state.nodes == (scenario.fallback ? 1 : 0): scenario.name - assert state.fetches == (scenario.fallback ? 1 : 0): scenario.name - def shouldEvaluate = (!scenario.readFailure || scenario.fallback) && !scenario.fetchFailure - assert state.evaluated == (shouldEvaluate ? [scenario.fallback ? 'latest main helper' : 'selected revision helper'] : []): scenario.name + assert state.nodes == 0: scenario.name + assert state.fetches == 0: scenario.name + def shouldEvaluate = !scenario.readFailure + assert state.evaluated == (shouldEvaluate ? ['selected revision helper'] : []): scenario.name count++ } } diff --git a/tests/python/ci/test_jenkins_helpers.py b/tests/python/ci/test_jenkins_helpers.py index f521d41303..d6653f8198 100644 --- a/tests/python/ci/test_jenkins_helpers.py +++ b/tests/python/ci/test_jenkins_helpers.py @@ -135,3 +135,178 @@ with pathlib.Path(os.environ["COMMAND_LOG"]).open("a") as out: assert build[0][4].startswith("-j") and int(build[0][4][2:]) >= 1 assert len(build[0]) == 5 assert build[1:] == configure[1:] + + [email protected] +def artifact_cli(tmp_path): + script_root = tmp_path / "ci/jenkins/scripts" + script_root.mkdir(parents=True) + for name in ("s3.py", "retry.sh"): + shutil.copy(JENKINS_SCRIPT_ROOT / name, script_root / name) + registry = { + "wheels": ["build/dist/*.whl", "build/dist/a.whl", "./build/dist/a.whl"], + "extra": ["build/exact.so", "build/dir", "build/missing.so", "build/no-match/*.so"], + } + (script_root.parent / "artifacts.py").write_text("ARTIFACTS = " + repr(registry)) + for name in ( + "build/dist/b space.whl", + "build/dist/a.whl", + "build/exact.so", + "build/dir/nested", + ): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("artifact") + binary_dir = tmp_path / "bin" + binary_dir.mkdir() + executable( + binary_dir / "aws", + """ +import json, os, pathlib, sys +with pathlib.Path(os.environ["AWS_LOG"]).open("a") as out: + out.write(json.dumps(sys.argv[1:]) + "\\n") +if sys.argv[-1] == ".": + pathlib.Path("downloaded file").write_text("downloaded") + print("download: s3://bucket/prefix/downloaded file to downloaded file") +""", + ) + log = tmp_path / "commands" + env = dict(os.environ, PATH=f"{binary_dir}:{os.environ['PATH']}", AWS_LOG=str(log)) + + def run(*options, action="upload"): + log.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + script_root / "s3.py", + "--action", + action, + "--bucket", + "bucket", + "--prefix", + "/prefix/", + *options, + ], + cwd=tmp_path, + env=env, + text=True, + capture_output=True, + ) + commands = ( + [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + ) + return result, commands + + return run + + +def test_artifact_upload_aliases_and_globs(artifact_cli): + expected = [ + ["s3", "cp", "--no-progress", "--recursive", "build/dir", "s3://bucket/prefix/build/dir"], + ["s3", "cp", "--no-progress", "build/dist/a.whl", "s3://bucket/prefix/build/dist/a.whl"], + [ + "s3", + "cp", + "--no-progress", + "build/dist/b space.whl", + "s3://bucket/prefix/build/dist/b space.whl", + ], + ["s3", "cp", "--no-progress", "build/exact.so", "s3://bucket/prefix/build/exact.so"], + ] + for flags in ( + ["--artifacts", "wheels", "extra"], + ["--artifacts", "extra", "--artifacts", "wheels"], + ["--bundle", "wheels", "--bundle", "extra"], + ["--bundle", "extra", "wheels"], + ): + result, commands = artifact_cli(*flags) + assert result.returncode == 0, result.stderr + assert commands == expected + assert "No artifacts matched pattern: build/no-match/*.so" in result.stderr + assert "The path doesn't exist: build/missing.so" in result.stderr + + +def test_artifact_paths_alias_and_unmatched_pattern(artifact_cli): + for flag in ("--paths", "--items"): + result, commands = artifact_cli(flag, "build/dist/*.whl", "./build/dist/a.whl") + assert result.returncode == 0, result.stderr + assert [command[-2] for command in commands] == [ + "build/dist/a.whl", + "build/dist/b space.whl", + ] + result, commands = artifact_cli(flag, "absent/*.whl") + assert result.returncode == 0 + assert commands == [] + assert "No artifacts matched pattern: absent/*.whl" in result.stderr + + [email protected]("artifacts_flag", ["--artifacts", "--bundle"]) [email protected]("paths_flag", ["--paths", "--items"]) +def test_artifact_flags_are_mutually_exclusive(artifact_cli, artifacts_flag, paths_flag): + result, commands = artifact_cli(artifacts_flag, "wheels", paths_flag, "build/exact.so") + assert result.returncode == 2 + assert "not allowed with argument" in result.stderr + assert commands == [] + + +def test_artifact_download_copies_whole_prefix(artifact_cli, tmp_path): + result, commands = artifact_cli(action="download") + assert result.returncode == 0, result.stderr + assert commands == [["s3", "cp", "--no-progress", "--recursive", "s3://bucket/prefix", "."]] + downloaded = tmp_path / "downloaded file" + assert downloaded.read_text() == "downloaded" + assert downloaded.stat().st_mode & 0o111 + + [email protected]("absolute", [False, True]) +def test_legacy_artifact_literal_paths(artifact_cli, tmp_path, absolute): + outside = tmp_path.parent / "outside.so" + outside.write_text("artifact") + path = str(outside) if absolute else "../outside.so" + result, commands = artifact_cli("--items", path) + assert result.returncode == 0, result.stderr + assert commands == [["s3", "cp", "--no-progress", path, "s3://bucket/prefix/" + path]] + + +def test_recursive_artifact_overlap(artifact_cli): + result, commands = artifact_cli("--paths", "build/dir", "build/dir/*", "build/dir/nested") + assert result.returncode == 0, result.stderr + assert commands == [ + ["s3", "cp", "--no-progress", "--recursive", "build/dir", "s3://bucket/prefix/build/dir"] + ] + + +def test_artifact_empty_directory_and_unknown_name(artifact_cli, tmp_path): + (tmp_path / "empty").mkdir() + result, commands = artifact_cli("--paths", "empty") + assert result.returncode != 0 + assert "Cannot upload empty folder" in result.stderr + assert commands == [] + result, commands = artifact_cli("--artifacts", "unknown") + assert result.returncode != 0 + assert "Unknown artifact" in result.stderr + assert commands == [] + + +def test_current_pipeline_artifact_options(artifact_cli, tmp_path): + # c14bc80 passes repeated --bundle names; the new Jenkinsfiles use --artifacts. + registry = load_s3().load_artifacts() + (tmp_path / "ci/jenkins/artifacts.py").write_text("ARTIFACTS = " + repr(registry)) + for entries in registry.values(): + for item in entries: + path = tmp_path / item + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("artifact") + for names in ( + ["tvm_lib", "cpptest"], + ["tvm_lib", "tvm_lib_gpu_extra", "cpptest"], + ["tvm_runtime"], + ): + old = [option for name in names for option in ("--bundle", name)] + old_result, old_commands = artifact_cli(*old) + new_result, new_commands = artifact_cli("--artifacts", *names) + assert old_result.returncode == new_result.returncode == 0 + assert old_commands == new_commands + assert [command[-2] for command in new_commands] == sorted( + {item for name in names for item in registry[name]} + ) diff --git a/tests/python/ci/test_jenkins_loader.py b/tests/python/ci/test_jenkins_loader.py index 2907c94b78..8d73c97642 100644 --- a/tests/python/ci/test_jenkins_loader.py +++ b/tests/python/ci/test_jenkins_loader.py @@ -14,19 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Exercise the native helper trust boundary and its isolated Git fetch.""" +"""Exercise the native helper trust boundary without workspace fallback.""" -import os import shutil import subprocess from pathlib import Path import pytest -from .test_utils import REPO_ROOT, TempGit +from .test_utils import REPO_ROOT PIPELINES = sorted((REPO_ROOT / "ci/jenkins/generated").glob("*_jenkinsfile.groovy")) -HELPER = "ci/jenkins/generated/common.groovy" def test_native_loader_trust_boundary(): @@ -37,70 +35,3 @@ def test_native_loader_trust_boundary(): [groovy, str(Path(__file__).with_name("jenkins_loader_test.groovy")), *PIPELINES], check=True, ) - - [email protected]("failure", [None, "fetch", "missing-helper"]) -def test_trusted_main_fetch(tmp_path, failure): - # Execute the actual loader shell against a local upstream. A Git shim only - # substitutes the transport URL; ref selection and object reads use real Git. - upstream = tmp_path / "upstream" - upstream.mkdir() - repo = TempGit(upstream) - repo.run("config", "user.name", "CI Test") - repo.run("config", "user.email", "[email protected]") - helper = upstream / HELPER - helper.parent.mkdir(parents=True) - helper.write_text("old main helper") - repo.run("add", ".") - repo.run("commit", "-m", "initial main") - repo.run("branch", "release") - helper.write_text("latest main helper") - repo.run("commit", "-am", "advance main") - if failure == "missing-helper": - helper.unlink() - repo.run("commit", "-am", "remove helper") - - checkout = tmp_path / "untrusted-checkout" - checkout.mkdir() - untrusted_helper = checkout / HELPER - untrusted_helper.parent.mkdir(parents=True) - untrusted_helper.write_text("rejected PR helper") - binary_dir = tmp_path / "bin" - binary_dir.mkdir() - git = binary_dir / "git" - git.write_text( - "#!/usr/bin/env python3\n" - "import os, sys\n" - "args = sys.argv[1:]\n" - "url = 'https://github.com/apache/tvm.git'\n" - "if 'fetch' in args:\n" - " assert url in args\n" - " assert '+refs/heads/main:refs/heads/trusted-main' in args\n" - " if os.environ.get('FAIL_FETCH'): sys.exit(7)\n" - " args[args.index(url)] = os.environ['TEST_UPSTREAM']\n" - "os.execv(os.environ['REAL_GIT'], ['git'] + args)\n" - ) - git.chmod(0o755) - env = dict( - os.environ, - PATH=f"{binary_dir}:{os.environ['PATH']}", - REAL_GIT=shutil.which("git"), - TEST_UPSTREAM=str(upstream), - FAIL_FETCH="yes" if failure == "fetch" else "", - CHANGE_TARGET="release", - ) - scripts = { - p.read_text().split("label: 'Read trusted helper from upstream main',")[1].split("'" * 3)[1] - for p in PIPELINES - } - assert len(scripts) == 1 - result = subprocess.run( - ["sh", "-c", scripts.pop()], cwd=checkout, env=env, text=True, capture_output=True - ) - if failure: - assert result.returncode != 0 - assert result.stdout == "" - else: - assert result.returncode == 0, result.stderr - assert result.stdout == "latest main helper" - assert untrusted_helper.read_text() == "rejected PR helper"
