This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 1b68589d69 ci: enable merge queue and run Miri only in the queue
(#11097)
1b68589d69 is described below
commit 1b68589d69848552f9b32866f71633fe92114596
Author: Dmitrii Blaginin <[email protected]>
AuthorDate: Fri Oct 2 05:28:35 2026 +0100
ci: enable merge queue and run Miri only in the queue (#11097)
- Closes https://github.com/apache/arrow-rs/issues/11045
- closes https://github.com/apache/arrow-rs/issues/8422
Add merge queue to arrow-rs and move miri to run there (and not on every
push)
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
.asf.yaml | 40 ++++-
.github/ci/paths.yaml | 106 +++++++++++++
.github/ci/scripts/check-ci-config.py | 270 ++++++++++++++++++++++++++++++++++
.github/ci/scripts/compute-changes.py | 76 ++++++++++
.github/workflows/README.md | 11 +-
.github/workflows/arrow.yml | 31 +---
.github/workflows/arrow_flight.yml | 22 +--
.github/workflows/audit.yml | 15 +-
.github/workflows/ci.yml | 147 ++++++++++++++++++
.github/workflows/dev.yml | 11 +-
.github/workflows/docs.yml | 11 +-
.github/workflows/integration.yml | 60 +++-----
.github/workflows/miri.yaml | 34 +----
.github/workflows/parquet.yml | 54 ++-----
.github/workflows/rust.yml | 11 +-
15 files changed, 691 insertions(+), 208 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 56980884a4..910b13f909 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -41,15 +41,47 @@ github:
protected_branches:
main:
required_status_checks:
- # require branches to be up-to-date before merging
- strict: true
- # don't require any jobs to pass
- contexts: []
+ # The merge queue tests each PR against the latest main.
+ strict: false
+ # This job aggregates all suites in .github/workflows/ci.yml.
+ contexts:
+ - "Required Checks"
required_pull_request_reviews:
required_approving_review_count: 1
+ # Raw GitHub Rulesets API payload, as in apache/datafusion-comet.
+ # Rulesets and classic branch protection apply together.
+ rulesets:
+ - name: Merge Queue
+ target: branch
+ enforcement: active
+ bypass_actors:
+ # apache/root (ASF Infra) can recover a blocked queue.
+ - actor_id: 118420
+ actor_type: Team
+ bypass_mode: always
+ conditions:
+ ref_name:
+ include:
+ - "~DEFAULT_BRANCH"
+ exclude: []
+ rules:
+ - type: merge_queue
+ parameters:
+ merge_method: SQUASH
+ grouping_strategy: ALLGREEN
+ # Limit concurrent CI builds while allowing speculative testing.
+ max_entries_to_build: 2
+ # Merge up to five already-green entries together. Each PR still
+ # gets its own squash commit; this does not batch CI builds.
+ min_entries_to_merge: 1
+ max_entries_to_merge: 5
+ min_entries_to_merge_wait_minutes: 0
+ # Allow for the full test suite and ASF runner scheduling delays.
+ check_response_timeout_minutes: 300
pull_requests:
# enable updating head branches of pull requests
allow_update_branch: true
+ allow_auto_merge: true
# publishes the content of the `asf-site` branch to
# https://arrow.apache.org/rust/
diff --git a/.github/ci/paths.yaml b/.github/ci/paths.yaml
new file mode 100644
index 0000000000..f89ca2fe2c
--- /dev/null
+++ b/.github/ci/paths.yaml
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Shared build inputs select all suites on pull requests.
+common:
+ - .asf.yaml
+ - .github/**
+ - .gitmodules
+ - .config/**
+ - Cargo.toml
+ - Cargo.lock
+ - rust-toolchain.toml
+ - testing
+ - parquet-testing
+ - format/**
+suites:
+ arrow:
+ - arrow-arith/**
+ - arrow-array/**
+ - arrow-buffer/**
+ - arrow-cast/**
+ - arrow-cmp/**
+ - arrow-csv/**
+ - arrow-data/**
+ - arrow-integration-test/**
+ - arrow-integration-testing/**
+ - arrow-ipc/**
+ - arrow-json/**
+ - arrow-avro/**
+ - arrow-ord/**
+ - arrow-row/**
+ - arrow-schema/**
+ - arrow-select/**
+ - arrow-string/**
+ - arrow/**
+ arrow-flight:
+ - arrow-array/**
+ - arrow-buffer/**
+ - arrow-cast/**
+ - arrow-data/**
+ - arrow-flight/**
+ - arrow-ipc/**
+ - arrow-schema/**
+ - arrow-select/**
+ parquet:
+ - arrow/**
+ - arrow-arith/**
+ - arrow-array/**
+ - arrow-buffer/**
+ - arrow-cast/**
+ - arrow-cmp/**
+ - arrow-csv/**
+ - arrow-data/**
+ - arrow-ipc/**
+ - arrow-json/**
+ - arrow-ord/**
+ - arrow-row/**
+ - arrow-schema/**
+ - arrow-select/**
+ - arrow-string/**
+ - parquet/**
+ - parquet_derive/**
+ - parquet_derive_test/**
+ - parquet-geospatial/**
+ - parquet-variant/**
+ - parquet-variant-compute/**
+ - parquet-variant-json/**
+ integration:
+ - arrow-arith/**
+ - arrow-array/**
+ - arrow-buffer/**
+ - arrow-cast/**
+ - arrow-cmp/**
+ - arrow-csv/**
+ - arrow-data/**
+ - arrow-flight/**
+ - arrow-integration-test/**
+ - arrow-integration-testing/**
+ - arrow-ipc/**
+ - arrow-json/**
+ - arrow-ord/**
+ - arrow-pyarrow-integration-testing/**
+ - arrow-pyarrow-testing/**
+ - arrow-pyarrow/**
+ - arrow-row/**
+ - arrow-schema/**
+ - arrow-select/**
+ - arrow-string/**
+ - arrow/**
+ audit:
+ - '**/Cargo.toml'
+ - '**/Cargo.lock'
diff --git a/.github/ci/scripts/check-ci-config.py
b/.github/ci/scripts/check-ci-config.py
new file mode 100644
index 0000000000..2648251b4f
--- /dev/null
+++ b/.github/ci/scripts/check-ci-config.py
@@ -0,0 +1,270 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Check CI configuration and routing policy, following Comet's structure.
+
+Run from the repository root:
+ python3 .github/ci/scripts/check-ci-config.py
+"""
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import tomllib
+import unittest
+
+spec = importlib.util.spec_from_file_location(
+ "compute_changes", Path(__file__).with_name("compute-changes.py")
+)
+changes = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(changes)
+
+ROOT = changes.ROOT
+ALWAYS = changes.ALWAYS
+load_yaml = changes.load_yaml
+
+# Only these jobs and shell steps should run when warming main's caches.
+CACHE_REFRESH_STEPS = {
+ "integration": {
+ "pyarrow-integration-test": {
+ "Setup Rust toolchain", "Upgrade pip and setuptools",
+ "Create virtualenv and install dependencies", "Build Python
extension",
+ },
+ },
+ "parquet": {"pyspark-integration-test": {"Install Python dependencies"}},
+}
+FULL_RUN = "${{ !inputs.cache-refresh-only }}"
+
+
+def validate_cache_refresh(ci, suite, workflow):
+ expected = "${{ needs.changes.outputs.cache-refresh-only == 'true' }}"
+ assert ci["jobs"][suite].get("with", {}).get("cache-refresh-only") ==
expected, f"{suite}: missing cache mode input"
+ cache_jobs = CACHE_REFRESH_STEPS[suite]
+ assert cache_jobs.keys() <= workflow["jobs"].keys(), f"{suite}: missing
cache writer"
+ for name, job in workflow["jobs"].items():
+ if name not in cache_jobs:
+ assert job.get("if") == FULL_RUN, f"{suite}/{name}: runs during
cache refresh"
+ continue
+ assert "if" not in job, f"{suite}/{name}: cache writer must run in
both modes"
+ run_steps = {step["name"]: step for step in job["steps"] if "run" in
step}
+ assert cache_jobs[name] <= run_steps.keys(), f"{suite}/{name}: missing
cache setup step"
+ for step in job["steps"]:
+ if "run" not in step or step["name"] in cache_jobs[name]:
+ assert "if" not in step, f"{suite}/{name}: cache setup must
run in both modes"
+ else:
+ assert step.get("if") == FULL_RUN,
f"{suite}/{name}/{step['name']}: runs during cache refresh"
+
+
+def validate_package_paths(root, suite, workflow, filters):
+ """Check literal Cargo package arguments, without interpreting shell
scripts."""
+ packages = {}
+ for manifest in root.glob("*/Cargo.toml"):
+ package = tomllib.loads(manifest.read_text()).get("package")
+ if package:
+ packages[package["name"]] =
manifest.parent.relative_to(root).as_posix()
+ for job in workflow["jobs"].values():
+ for step in job.get("steps", []):
+ for command in re.findall(r"\bcargo\b[^\n]*", step.get("run", "")):
+ for name in
re.findall(r"(?:^|\s)(?:-p|--package)(?:\s+|=)[\"']?([\w-]+)", command):
+ assert name in packages, f"{suite}: unknown local package
{name}"
+ path = f"{packages[name]}/src/lib.rs"
+ selected = changes.select_suites("pull_request", [path],
filters)
+ assert suite in selected, f"{suite}: changes to {name} do
not select its tests"
+
+
+def validate_config(root=ROOT):
+ """Catch stale check names, omitted suites, and workflow-level path
filters."""
+ workflows = root / ".github/workflows"
+ ci = load_yaml(workflows / "ci.yml")
+ filters = load_yaml(root / ".github/ci/paths.yaml")
+ jobs = ci["jobs"]
+ suites = set(filters["suites"]) | ALWAYS | {"miri"}
+ assert not (set(filters["suites"]) & (ALWAYS | {"miri"})), "Unexpected
filtered suite"
+ assert set(jobs) == suites | {"changes", "required-checks"}, "CI suites
and jobs differ"
+ # PyYAML's YAML 1.1 loader treats the key 'on' as the boolean True.
+ events = ci[True]
+ assert jobs["changes"]["outputs"].get("cache-refresh-only") == "${{
steps.select.outputs.cache-refresh-only }}", "Missing cache mode output"
+ assert set(events) == {"pull_request", "push", "merge_group"}, "Unexpected
CI triggers"
+ assert events["pull_request"] is None, "Required CI must run on every PR"
+ assert events["merge_group"] == {"types": ["checks_requested"]}, "Missing
queue trigger"
+ assert events["push"] == {"branches": ["main"]}, "Only main pushes should
run CI"
+
+ gate = jobs["required-checks"]
+ assert set(gate["needs"]) == set(jobs) - {"required-checks"}, "Required
Checks omits a job"
+ assert gate["if"] == "always()", "Required Checks must run after failures
and cancellations"
+ asf = load_yaml(root / ".asf.yaml")["github"]
+ checks = asf["protected_branches"]["main"]["required_status_checks"]
+ assert checks["contexts"] == [gate["name"]], "Required check name does not
match .asf.yaml"
+ assert checks["strict"] is False, "The queue handles up-to-date checks"
+
+ called = set()
+ for suite in suites:
+ job = jobs[suite]
+ assert job["needs"] == "changes", f"{suite}: missing suite selection
dependency"
+ condition = f"contains(fromJSON(needs.changes.outputs.suites),
'{suite}')"
+ assert job["if"] == condition, f"{suite}: unexpected routing condition"
+ path = root / job["uses"]
+ workflow = load_yaml(path)
+ call = None
+ if suite in CACHE_REFRESH_STEPS:
+ call = {"inputs": {"cache-refresh-only": {
+ "description": "Populate caches without running tests or
linters",
+ "type": "boolean", "default": False,
+ }}}
+ validate_cache_refresh(ci, suite, workflow)
+ assert workflow[True] == {"workflow_call": call}, f"{suite}: duplicate
or missing triggers/inputs"
+ assert "concurrency" not in workflow, f"{suite}: concurrency is owned
by ci.yml"
+ if suite in filters["suites"]:
+ validate_package_paths(root, suite, workflow, filters)
+ called.add(path.resolve())
+ reusable = {
+ path.resolve()
+ for path in workflows.iterdir()
+ if path.suffix in {".yml", ".yaml"}
+ and "workflow_call" in load_yaml(path).get(True, {})
+ }
+ assert called == reusable, "A reusable workflow is missing from Required
Checks"
+ miri = load_yaml(workflows / "miri.yaml")["jobs"]["miri-checks"]
+ assert "if" not in miri, "Miri event routing belongs in compute-changes.py"
+ print("CI configuration is valid")
+
+
+class RoutingTests(unittest.TestCase):
+ def setUp(self):
+ self.filters = changes.load_yaml(changes.ROOT /
".github/ci/paths.yaml")
+ self.always = {"dev", "rust", "docs"}
+ self.all_suites = self.always | {
+ "arrow", "arrow-flight", "parquet", "integration", "audit",
+ }
+
+ def select(self, event, paths):
+ return set(changes.select_suites(event, paths, self.filters))
+
+ def test_event_policy(self):
+ for paths in [[], ["README.md"], ["arrow-buffer/src/lib.rs"]]:
+ with self.subTest(paths=paths):
+ self.assertEqual(self.select("merge_group", paths),
self.all_suites | {"miri"})
+ self.assertEqual(self.select("push", paths), {"docs",
"integration", "parquet"})
+ self.assertNotIn("miri", self.select("pull_request", paths))
+ with self.assertRaises(ValueError):
+ self.select("pull_request_target", [])
+
+ def test_selector_outputs(self):
+ for event, expected in [("push", {"docs", "integration", "parquet"}),
+ ("merge_group", self.all_suites | {"miri"}),
+ ("pull_request", self.always)]:
+ with self.subTest(event=event), tempfile.TemporaryDirectory() as
directory:
+ output = Path(directory) / "output"
+ env = dict(os.environ, GITHUB_EVENT_NAME=event,
GITHUB_OUTPUT=str(output),
+ PR_BASE_SHA="HEAD", PR_HEAD_SHA="HEAD")
+ subprocess.run([sys.executable, str(ROOT /
".github/ci/scripts/compute-changes.py")],
+ env=env, check=True, capture_output=True)
+ values = dict(line.split("=", 1) for line in
output.read_text().splitlines())
+ self.assertEqual(set(json.loads(values["suites"])), expected)
+ self.assertEqual(json.loads(values["cache-refresh-only"]),
event == "push")
+
+ def test_pr_path_routing(self):
+ cases = [
+ ([], set()),
+ (["README.md"], set()),
+ (["arrow-buffer/src/lib.rs"], {"arrow", "arrow-flight", "parquet",
"integration"}),
+ (["arrow-avro/src/lib.rs"], {"arrow"}),
+ (["arrow-flight/src/lib.rs"], {"arrow-flight", "integration"}),
+ (["arrow-pyarrow/src/lib.rs"], {"integration"}),
+ (["arrow-integration-testing/src/lib.rs"], {"arrow",
"integration"}),
+ (["arrow-pyarrow-testing/src/lib.rs"], {"integration"}),
+ (["parquet/src/lib.rs"], {"parquet"}),
+ (["parquet_derive/src/lib.rs"], {"parquet"}),
+ (["parquet_derive_test/src/lib.rs"], {"parquet"}),
+ (["parquet-variant-json/src/lib.rs"], {"parquet"}),
+ (["parquet-geospatial/src/lib.rs"], {"parquet"}),
+ (["arrow-row/Cargo.toml"], {"arrow", "parquet", "integration",
"audit"}),
+ (["some/nested/Cargo.lock"], {"audit"}),
+ (["parquet-variant/src/lib.rs"], {"parquet"}),
+ (["parquet-variant-compute/src/lib.rs"], {"parquet"}),
+ (["arrow-arith/src/lib.rs"], {"arrow", "parquet", "integration"}),
+ (["arrow-ord/src/lib.rs"], {"arrow", "parquet", "integration"}),
+ (["arrow-string/src/lib.rs"], {"arrow", "parquet", "integration"}),
+ # A move or deletion must still select the affected suites.
+ (["arrow-cmp/old.rs", "parquet-geospatial/new.rs"], {"arrow",
"parquet", "integration"}),
+ (["arrow-cmp/file\nwith-newline.rs"], {"arrow", "parquet",
"integration"}),
+ ]
+ for paths, expected in cases:
+ with self.subTest(paths=paths):
+ self.assertEqual(self.select("pull_request", paths),
self.always | expected)
+
+ def test_shared_build_inputs(self):
+ for path in [
+ ".asf.yaml", ".github/workflows/ci.yml", ".github/ci/paths.yaml",
+ ".github/actions/setup-builder/action.yaml", ".gitmodules",
+ "Cargo.toml", "Cargo.lock", "rust-toolchain.toml",
+ "testing", "parquet-testing", "format/Flight.proto",
"format/FlightSql.proto",
+ ".config/nextest.toml",
+ ]:
+ with self.subTest(path=path):
+ self.assertEqual(self.select("pull_request", [path]),
self.all_suites)
+
+
+class ConfigurationTests(unittest.TestCase):
+ def test_current_configuration(self):
+ with contextlib.redirect_stdout(io.StringIO()):
+ validate_config()
+
+ def test_configuration_regressions(self):
+ cases = [
+ (".asf.yaml", '"Required Checks"', '"Missing Check"'),
+ (".github/workflows/ci.yml", " - miri\n", ""),
+ (".github/workflows/ci.yml", " pull_request:\n", "
pull_request:\n paths: ['arrow/**']\n"),
+ (".github/workflows/ci.yml", " if: always()", " if:
success()"),
+ (".github/workflows/miri.yaml", " name: MIRI\n", " name:
MIRI\n if: github.event_name == 'merge_group'\n"),
+ (".github/workflows/arrow.yml", " workflow_call:", "
workflow_call:\n pull_request:"),
+ (".github/ci/paths.yaml", " - arrow-integration-testing/**\n",
""),
+ (".github/workflows/ci.yml", " cache-refresh-only: ${{
steps.select.outputs.cache-refresh-only }}\n", ""),
+ (".github/workflows/ci.yml", " with:\n cache-refresh-only:
${{ needs.changes.outputs.cache-refresh-only == 'true' }}\n", ""),
+ (".github/workflows/parquet.yml", "\n if: ${{
!inputs.cache-refresh-only }}\n", "\n"),
+ (".github/workflows/integration.yml", "\n if: ${{
!inputs.cache-refresh-only }}\n", "\n"),
+ (".github/workflows/parquet.yml", " if: ${{
!inputs.cache-refresh-only }}\n", ""),
+ (".github/workflows/integration.yml", " if: ${{
!inputs.cache-refresh-only }}\n", ""),
+ (".github/workflows/integration.yml", " - name: Build Python
extension\n", " - name: Build Python extension\n if: ${{
!inputs.cache-refresh-only }}\n"),
+ ]
+ for filename, before, after in cases:
+ with self.subTest(filename=filename, before=before),
tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ shutil.copytree(changes.ROOT / ".github", root / ".github")
+ shutil.copy(changes.ROOT / ".asf.yaml", root / ".asf.yaml")
+ for manifest in changes.ROOT.glob("*/Cargo.toml"):
+ target = root / manifest.relative_to(changes.ROOT)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(manifest, target)
+ path = root / filename
+ original = path.read_text()
+ self.assertIn(before, original)
+ path.write_text(original.replace(before, after))
+ with self.assertRaises(AssertionError):
+ validate_config(root)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/ci/scripts/compute-changes.py
b/.github/ci/scripts/compute-changes.py
new file mode 100644
index 0000000000..0b8706206c
--- /dev/null
+++ b/.github/ci/scripts/compute-changes.py
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Select CI suites from the event and changed PR paths."""
+
+import fnmatch
+import json
+import os
+from pathlib import Path
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[3]
+ALWAYS = {"dev", "rust", "docs"}
+
+
+def load_yaml(path):
+ import yaml
+
+ return yaml.safe_load(path.read_text())
+
+
+def select_suites(event, paths, filters):
+ suites = set(filters["suites"]) | ALWAYS
+ if event == "merge_group":
+ return sorted(suites | {"miri"})
+ if event == "push":
+ # The queue already tested the merge result. Publish docs and populate
+ # main's caches through the reusable workflows' cache-refresh-only
mode.
+ return ["docs", "integration", "parquet"]
+ if event != "pull_request":
+ raise ValueError(f"Unsupported CI event: {event}")
+
+ def matches(patterns):
+ return any(fnmatch.fnmatchcase(path, pattern) for path in paths for
pattern in patterns)
+
+ if matches(filters["common"]):
+ return sorted(suites)
+ return sorted(ALWAYS | {name for name, patterns in
filters["suites"].items() if matches(patterns)})
+
+
+def main():
+ event = os.environ["GITHUB_EVENT_NAME"]
+ paths = []
+ if event == "pull_request":
+ base = os.environ["PR_BASE_SHA"]
+ head = os.environ["PR_HEAD_SHA"]
+ # Full checkout history includes both PR parents. Disable rename
+ # detection so moving a file selects both its old and new suites.
+ changed = subprocess.check_output(
+ ["git", "diff", "--no-renames", "--name-only", "-z",
f"{base}...{head}", "--"],
+ cwd=ROOT,
+ )
+ paths = changed.decode("utf-8",
errors="surrogateescape").split("\0")[:-1]
+ suites = select_suites(event, paths, load_yaml(ROOT /
".github/ci/paths.yaml"))
+ with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+ output.write(f"suites={json.dumps(suites)}\n")
+ output.write(f"cache-refresh-only={json.dumps(event == 'push')}\n")
+ print(f"Selected suites: {', '.join(suites)}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index cfdd6d143c..d5c5d915c6 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -17,10 +17,7 @@
under the License.
-->
-The CI is structured so most tests are run in specific workflows:
-`arrow.yml` for `arrow`, `parquet.yml` for `parquet` and the three
-`parquet-variant*` crates, and so on.
-
-The basic idea is to run all tests on pushes to main (to ensure we
-keep main green) but run only the individual workflows on PRs that
-change files that could affect them.
+CI runs a subset of checks on pull requests, selected by the changed files.
+The merge queue runs the full test suite against the proposed merge result.
+Pushes to `main` populate caches and publish documentation without repeating
tests.
+A single required status check combines the results and blocks merging on
failure.
diff --git a/.github/workflows/arrow.yml b/.github/workflows/arrow.yml
index 71d4bda10f..a23ac9c214 100644
--- a/.github/workflows/arrow.yml
+++ b/.github/workflows/arrow.yml
@@ -18,36 +18,9 @@
# tests for arrow crate
name: arrow
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- # always trigger
- push:
- branches:
- - main
- pull_request:
- paths:
- - .github/**
- - rust-toolchain.toml
- - arrow-arith/**
- - arrow-array/**
- - arrow-buffer/**
- - arrow-cast/**
- - arrow-cmp/**
- - arrow-csv/**
- - arrow-data/**
- - arrow-integration-test/**
- - arrow-ipc/**
- - arrow-json/**
- - arrow-avro/**
- - arrow-ord/**
- - arrow-row/**
- - arrow-schema/**
- - arrow-select/**
- - arrow-string/**
- - arrow/**
+ workflow_call:
permissions:
contents: read
diff --git a/.github/workflows/arrow_flight.yml
b/.github/workflows/arrow_flight.yml
index 1e158dabe3..f11b0d1b41 100644
--- a/.github/workflows/arrow_flight.yml
+++ b/.github/workflows/arrow_flight.yml
@@ -19,27 +19,9 @@
# tests for arrow_flight crate
name: arrow_flight
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs that touch certain files and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
- paths:
- - rust-toolchain.toml
- - arrow-array/**
- - arrow-buffer/**
- - arrow-cast/**
- - arrow-data/**
- - arrow-flight/**
- - arrow-ipc/**
- - arrow-schema/**
- - arrow-select/**
- - .github/**
+ workflow_call:
permissions:
contents: read
diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml
index aec6c9db53..7a6b183d21 100644
--- a/.github/workflows/audit.yml
+++ b/.github/workflows/audit.yml
@@ -17,20 +17,9 @@
name: audit
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs that touch certain files and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
- paths:
- - '**/Cargo.toml'
- - '**/Cargo.lock'
- - rust-toolchain.toml
+ workflow_call:
permissions:
contents: read
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000000..dcd624568c
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,147 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: CI
+
+on:
+ pull_request:
+ merge_group:
+ types: [checks_requested]
+ push:
+ branches: [main]
+
+concurrency:
+ group: ${{ github.repository }}-${{ github.event_name }}-${{
github.event.pull_request.number || github.sha }}-${{ github.workflow }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ changes:
+ name: Select and validate CI suites
+ runs-on: ubuntu-latest
+ outputs:
+ suites: ${{ steps.select.outputs.suites }}
+ cache-refresh-only: ${{ steps.select.outputs.cache-refresh-only }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ # PR routing uses the merge base, including changes from all commits.
+ fetch-depth: 0
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
+ with:
+ python-version: '3.12'
+ - name: Install CI configuration dependency
+ run: |
+ for attempt in 1 2 3; do
+ python3 -m pip install PyYAML==6.0.3 && break
+ if [ "$attempt" -eq 3 ]; then exit 1; fi
+ sleep 5
+ done
+ - name: Validate CI configuration and routing
+ run: python3 .github/ci/scripts/check-ci-config.py
+ - name: Select suites
+ id: select
+ env:
+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: python3 .github/ci/scripts/compute-changes.py
+
+ arrow:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'arrow')
+ uses: ./.github/workflows/arrow.yml
+
+ arrow-flight:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'arrow-flight')
+ uses: ./.github/workflows/arrow_flight.yml
+
+ parquet:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'parquet')
+ uses: ./.github/workflows/parquet.yml
+ with:
+ cache-refresh-only: ${{ needs.changes.outputs.cache-refresh-only ==
'true' }}
+
+ integration:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'integration')
+ uses: ./.github/workflows/integration.yml
+ with:
+ cache-refresh-only: ${{ needs.changes.outputs.cache-refresh-only ==
'true' }}
+
+ audit:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'audit')
+ uses: ./.github/workflows/audit.yml
+
+ dev:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'dev')
+ uses: ./.github/workflows/dev.yml
+
+ rust:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'rust')
+ uses: ./.github/workflows/rust.yml
+
+ docs:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'docs')
+ # Only the deploy job in docs.yml requests write access, on push to main.
+ permissions:
+ contents: write
+ uses: ./.github/workflows/docs.yml
+
+ miri:
+ needs: changes
+ if: contains(fromJSON(needs.changes.outputs.suites), 'miri')
+ uses: ./.github/workflows/miri.yaml
+
+ required-checks:
+ # Like datafusion-comet: one stable check name even when a reusable
+ # workflow is skipped (its individual check names would be different).
+ name: Required Checks
+ if: always()
+ permissions: {}
+ needs:
+ - changes
+ - arrow
+ - arrow-flight
+ - parquet
+ - integration
+ - audit
+ - dev
+ - rust
+ - docs
+ - miri
+ runs-on: ubuntu-slim
+ steps:
+ - name: Summarize suite results
+ env:
+ NEEDS: ${{ toJSON(needs) }}
+ run: echo "$NEEDS"
+ # Path-filtered suites and Miri outside the queue are intentionally
+ # skipped. A failed/cancelled selector or suite must still block merging.
+ - name: Fail if any suite failed or was cancelled
+ if: needs.changes.result != 'success' || contains(needs.*.result,
'failure') || contains(needs.*.result, 'cancelled')
+ run: |
+ echo "::error::One or more CI suites did not succeed. See the
results above."
+ exit 1
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 6b548e5af2..786f3ded90 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -17,16 +17,9 @@
name: dev
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
+ workflow_call:
permissions:
contents: read
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 54f5387aa3..8b8753b15b 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -17,16 +17,9 @@
name: docs
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
+ workflow_call:
jobs:
# test doc links still work
diff --git a/.github/workflows/integration.yml
b/.github/workflows/integration.yml
index 62a4b19f58..658081db89 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -17,45 +17,21 @@
name: integration
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs that touch certain files and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
- paths:
- - .github/**
- - rust-toolchain.toml
- - arrow-arith/**
- - arrow-array/**
- - arrow-buffer/**
- - arrow-cast/**
- - arrow-cmp/**
- - arrow-csv/**
- - arrow-data/**
- - arrow-flight/**
- - arrow-integration-test/**
- - arrow-integration-testing/**
- - arrow-ipc/**
- - arrow-json/**
- - arrow-ord/**
- - arrow-pyarrow-integration-testing/**
- - arrow-pyarrow/**
- - arrow-row/**
- - arrow-schema/**
- - arrow-select/**
- - arrow-string/**
- - arrow/**
+ workflow_call:
+ inputs:
+ cache-refresh-only:
+ description: Populate caches without running tests or linters
+ type: boolean
+ default: false
permissions:
contents: read
jobs:
integration:
+ if: ${{ !inputs.cache-refresh-only }}
name: Archery test With other arrows
runs-on: ubuntu-latest
container:
@@ -198,7 +174,8 @@ jobs:
matrix:
rust: [stable]
# PyArrow 15 was the first version to introduce StringView/BinaryView
support
- pyarrow: ["15", "16", "17"]
+ # The cache keys are shared across these versions; one build warms
them.
+ pyarrow: ${{ inputs.cache-refresh-only && fromJSON('["17"]') ||
fromJSON('["15", "16", "17"]') }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
@@ -212,13 +189,15 @@ jobs:
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: /home/runner/.cargo
- key: cargo-maturin-cache-
+ key: cargo-maturin-cache-${{ hashFiles('**/Cargo.toml',
'rust-toolchain.toml') }}
+ restore-keys: cargo-maturin-cache-
- name: Cache Rust dependencies
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
- path: /home/runner/target
+ path: arrow-pyarrow-integration-testing/target
# this key is not equal because maturin uses different compilation
flags.
- key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{
matrix.rust }}-
+ key: ${{ runner.os }}-${{ runner.arch }}-target-maturin-cache-${{
matrix.rust }}-${{ hashFiles('**/Cargo.toml', 'rust-toolchain.toml') }}
+ restore-keys: ${{ runner.os }}-${{ runner.arch
}}-target-maturin-cache-${{ matrix.rust }}-
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
with:
python-version: '3.8'
@@ -230,6 +209,7 @@ jobs:
source venv/bin/activate
pip install maturin toml pytest pytz pyarrow==${{ matrix.pyarrow }}
- name: Run Rust tests
+ if: ${{ !inputs.cache-refresh-only }}
run: |
source venv/bin/activate
# arrow-pyarrow's own tests are feature-gated, so the workspace-wide
`cargo test` in
@@ -239,6 +219,7 @@ jobs:
cd arrow-pyarrow-testing
cargo test
- name: Clippy arrow-pyarrow
+ if: ${{ !inputs.cache-refresh-only }}
run: |
source venv/bin/activate
# Here rather than in arrow.yml's clippy job: that one runs in a
@@ -248,9 +229,14 @@ jobs:
# default features, so the --no-default-features variant arrow.yml
# also runs would be this same command.
cargo clippy -p arrow-pyarrow --all-features -- -D
unused_crate_dependencies
- - name: Run Python tests
+ - name: Build Python extension
run: |
source venv/bin/activate
cd arrow-pyarrow-integration-testing
maturin develop
+ - name: Run Python tests
+ if: ${{ !inputs.cache-refresh-only }}
+ run: |
+ source venv/bin/activate
+ cd arrow-pyarrow-integration-testing
pytest -v .
diff --git a/.github/workflows/miri.yaml b/.github/workflows/miri.yaml
index a5133388cd..3e57aa142d 100644
--- a/.github/workflows/miri.yaml
+++ b/.github/workflows/miri.yaml
@@ -17,39 +17,9 @@
name: miri
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs that touch certain files and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
- paths:
- - .config/nextest.toml
- - Cargo.toml
- - Cargo.lock
- - .github/**
- - rust-toolchain.toml
- - arrow/**
- - arrow-arith/**
- - arrow-array/**
- - arrow-avro/**
- - arrow-buffer/**
- - arrow-cast/**
- - arrow-csv/**
- - arrow-data/**
- - arrow-ipc/**
- - arrow-json/**
- - arrow-cmp/**
- - arrow-ord/**
- - arrow-row/**
- - arrow-schema/**
- - arrow-select/**
- - arrow-string/**
- - parquet/**
+ workflow_call:
permissions:
contents: read
diff --git a/.github/workflows/parquet.yml b/.github/workflows/parquet.yml
index c1c8c6704c..380f456f4d 100644
--- a/.github/workflows/parquet.yml
+++ b/.github/workflows/parquet.yml
@@ -19,46 +19,14 @@
# tests for parquet and related crates
name: "parquet"
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs that touch certain files and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
- paths:
- - Cargo.toml
- - Cargo.lock
- - rust-toolchain.toml
- - .gitmodules
- - testing
- - parquet-testing
- - arrow/**
- - arrow-arith/**
- - arrow-array/**
- - arrow-buffer/**
- - arrow-cast/**
- - arrow-cmp/**
- - arrow-csv/**
- - arrow-data/**
- - arrow-ipc/**
- - arrow-json/**
- - arrow-ord/**
- - arrow-row/**
- - arrow-schema/**
- - arrow-select/**
- - arrow-string/**
- - parquet/**
- - parquet_derive/**
- - parquet_derive_test/**
- - parquet-geospatial/**
- - parquet-variant/**
- - parquet-variant-compute/**
- - parquet-variant-json/**
- - .github/**
+ workflow_call:
+ inputs:
+ cache-refresh-only:
+ description: Populate caches without running tests or linters
+ type: boolean
+ default: false
permissions:
contents: read
@@ -66,6 +34,7 @@ permissions:
jobs:
# test the crate
linux-test:
+ if: ${{ !inputs.cache-refresh-only }}
name: Test
runs-on: ubuntu-latest
container:
@@ -100,6 +69,7 @@ jobs:
# test compilation
linux-features:
+ if: ${{ !inputs.cache-refresh-only }}
name: Check Compilation
runs-on: ubuntu-latest
container:
@@ -163,6 +133,7 @@ jobs:
# test the parquet crate builds against wasm32 in stable rust
wasm32-build:
+ if: ${{ !inputs.cache-refresh-only }}
name: Build wasm32
runs-on: ubuntu-latest
container:
@@ -202,24 +173,29 @@ jobs:
cd parquet/pytest
pip install -r requirements.txt
- name: Black check the test files
+ if: ${{ !inputs.cache-refresh-only }}
run: |
cd parquet/pytest
black --check *.py --verbose
- name: Setup Rust toolchain
+ if: ${{ !inputs.cache-refresh-only }}
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
- name: Install binary for checking
+ if: ${{ !inputs.cache-refresh-only }}
run: |
cargo install --path parquet --bin parquet-show-bloom-filter
--features=cli
cargo install --path parquet --bin parquet-fromcsv
--features=arrow,cli
cargo install --path parquet --bin parquet-index --features=cli
- name: Run pytest
+ if: ${{ !inputs.cache-refresh-only }}
run: |
cd parquet/pytest
pytest -v
clippy:
+ if: ${{ !inputs.cache-refresh-only }}
name: Clippy
runs-on: ubuntu-latest
container:
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 935548deda..9b1b5d3872 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -18,16 +18,9 @@
# workspace wide tests
name: rust
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-# trigger for all PRs and changes to main
+# Triggered by ci.yml, which owns event routing and concurrency.
on:
- push:
- branches:
- - main
- pull_request:
+ workflow_call:
permissions:
contents: read
