kumarUjjawal commented on code in PR #23414: URL: https://github.com/apache/datafusion/pull/23414#discussion_r3552661986
########## xtask/src/ci_workflow_rust.rs: ########## @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::ci_jobs::{JOBS, JobInfo}; +use crate::ci_workflows::WorkflowInfo; + +const VERIFY_CLEAN_JOB_NAME: &str = "verify-clean"; +const SKIPPED_JOBS: &[&str] = &[ + // TODO: Enable these after local setup is fixed. + "linux-wasm-pack", + "verify-benchmark-results", + "sqllogictest-postgres", +]; + +pub(crate) const WORKFLOW_RUST: WorkflowInfo = WorkflowInfo { + name: "rust", + help_usage: "ci workflow rust", + help_description: "Run Rust CI jobs locally", + jobs, +}; + +fn jobs() -> Vec<&'static JobInfo> { + let verify_clean = JOBS + .iter() + .find(|job| job.name == VERIFY_CLEAN_JOB_NAME) + .unwrap_or_else(|| panic!("missing `{VERIFY_CLEAN_JOB_NAME}` job")); + let mut jobs = vec![verify_clean]; Review Comment: fails immediately if we have there are any uncommitted changes before it runs any CI check. Was this intentional? ########## xtask/src/step_runners.rs: ########## @@ -0,0 +1,701 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Result, Xtask}; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; +use std::process::Command; + +// --------------------------------------------- +// Leaf step runner implementations and utilities. +// --------------------------------------------- + +impl Xtask { + // Step runner entry point for `cargo xtask ci step fmt ...`. + // See the corresponding `fmt` StepInfo for supported args. + pub(crate) fn run_fmt(&self, args: &[String]) -> Result<()> { + self.script("ci/scripts/rust_fmt.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step clippy ...`. + // See the corresponding `clippy` StepInfo for supported args. + pub(crate) fn run_clippy(&self, args: &[String]) -> Result<()> { + self.command("rustup") + .args(["component", "add", "clippy"]) + .run()?; + self.script("ci/scripts/rust_clippy.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-check ...`. + // See the corresponding `rust-check` StepInfo for supported args. + pub(crate) fn run_rust_check(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("workspace"); + if args.len() > 1 { + return Err("usage: cargo xtask ci step rust-check [target]".to_string()); + } + + match target { + "workspace" => self + .command("cargo") + .args([ + "check", + "--profile", + "ci", + "--workspace", + "--all-targets", + "--features", + "integration-tests", + "--locked", + ]) + .run(), + "datafusion-common" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-common", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-common", + ]) + .run() + } + "datafusion-substrait" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-substrait", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + "--features=physical", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + "--features=protoc", + ]) + .run() + } + "datafusion-proto" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-proto", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-proto", + ]) + .run()?; + for feature in ["json", "parquet", "avro"] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-proto", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + "datafusion" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ]) + .run()?; + for feature in [ + "nested_expressions", + "array_expressions", + "avro", + "backtrace", + "compression", + "crypto_expressions", + "datetime_expressions", + "encoding_expressions", + "force_hash_collisions", + "math_expressions", + "parquet", + "regex_expressions", + "recursive_protection", + "serde", + "sql", + "string_expressions", + "unicode_expressions", + "parquet_encryption", + ] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + "datafusion-functions" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-functions", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-functions", + ]) + .run()?; + for feature in [ + "crypto_expressions", + "datetime_expressions", + "encoding_expressions", + "math_expressions", + "regex_expressions", + "string_expressions", + "unicode_expressions", + ] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-functions", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + unknown => Err(format!("unknown rust-check target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step rust-test ...`. + // See the corresponding `rust-test` StepInfo for supported args. + pub(crate) fn run_rust_test(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("workspace"); + if args.len() > 1 { + return Err( + "usage: cargo xtask ci step rust-test [workspace|datafusion-cli|datafusion-ffi]" + .to_string(), + ); + } + + match target { + "workspace" => self + .command("cargo") + .args([ + "test", + "--profile", + "ci", + "--exclude", + "datafusion-examples", + "--exclude", + "ffi_example_table_provider", + "--exclude", + "datafusion-cli", + "--workspace", + "--lib", + "--tests", + "--bins", + "--features", + "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait", + ]) + .env("RUST_BACKTRACE", "1") + .run(), + "datafusion-cli" => self + .command("cargo") + .args([ + "test", + "--features", + "backtrace", + "--profile", + "ci", + "-p", + "datafusion-cli", + "--lib", + "--tests", + "--bins", + ]) + .env("RUST_BACKTRACE", "1") + .env("AWS_ENDPOINT", "http://127.0.0.1:9000";) + .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin") + .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword") + .env("TEST_STORAGE_INTEGRATION", "1") + .env("AWS_ALLOW_HTTP", "true") + .run(), + "datafusion-ffi" => self + .command("cargo") + .args([ + "test", + "--profile", + "ci", + "-p", + "datafusion-ffi", + "--lib", + "--tests", + "--features", + "integration-tests", + ]) + .run(), + unknown => Err(format!("unknown rust-test target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step rust-doctest ...`. + // See the corresponding `rust-doctest` StepInfo for supported args. + pub(crate) fn run_rust_doctest(&self, _args: &[String]) -> Result<()> { + self.command("cargo") + .args([ + "test", + "--profile", + "ci", + "--doc", + "--features", + "avro,json", + ]) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-doc ...`. + // See the corresponding `rust-doc` StepInfo for supported args. + pub(crate) fn run_rust_doc(&self, _args: &[String]) -> Result<()> { + self.script("ci/scripts/rust_docs.sh").run() + } + + // Step runner entry point for `cargo xtask ci step rust-examples ...`. + // See the corresponding `rust-examples` StepInfo for supported args. + pub(crate) fn run_rust_examples(&self, _args: &[String]) -> Result<()> { + self.command("cargo") + .args(["run", "--profile", "ci", "--example", "sql"]) + .run()?; + self.script("ci/scripts/rust_example.sh").run() + } + + // Step runner entry point for `cargo xtask ci step rust-wasm ...`. + // See the corresponding `rust-wasm` StepInfo for supported args. + pub(crate) fn run_rust_wasm(&self, _args: &[String]) -> Result<()> { + let wasm_dir = self.root.join("datafusion/wasmtest"); + let rustflags = r#"--cfg getrandom_backend="wasm_js" -C debuginfo=none"#; + self.command("wasm-pack") + .args(["test", "--headless", "--firefox"]) + .current_dir(&wasm_dir) + .env("RUSTFLAGS", rustflags) + .run()?; + + let chrome_driver = format!( + "{}/chromedriver", + env::var("CHROMEWEBDRIVER").unwrap_or_default() + ); + self.command("wasm-pack") + .args([ + "test", + "--headless", + "--chrome", + "--chromedriver", + chrome_driver.as_str(), + ]) + .current_dir(&wasm_dir) + .env("RUSTFLAGS", rustflags) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-benchmark-results ...`. + // See the corresponding `rust-benchmark-results` StepInfo for supported args. + // + // Generate TPCH data, then run checks that require `TPCH_DATA`: + // - benchmark Rust tests for TPCH plan round-trip coverage + // - sqllogictests for TPCH plans and query answers + pub(crate) fn run_rust_benchmark_results(&self, _args: &[String]) -> Result<()> { + let tpch_data = self + .root + .join("datafusion/sqllogictest/test_files/tpch/data"); + fs::create_dir_all(&tpch_data).map_err(|error| { + format!("failed to create {}: {error}", tpch_data.display()) + })?; + + self.command("git") Review Comment: if the step fails partway through and I re-run, git clone immediately fails with "destination path already exists". I'd have to manually rm -rf tpch-dbgen before retrying. ########## xtask/src/step_runners.rs: ########## @@ -0,0 +1,701 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Result, Xtask}; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; +use std::process::Command; + +// --------------------------------------------- +// Leaf step runner implementations and utilities. +// --------------------------------------------- + +impl Xtask { + // Step runner entry point for `cargo xtask ci step fmt ...`. + // See the corresponding `fmt` StepInfo for supported args. + pub(crate) fn run_fmt(&self, args: &[String]) -> Result<()> { + self.script("ci/scripts/rust_fmt.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step clippy ...`. + // See the corresponding `clippy` StepInfo for supported args. + pub(crate) fn run_clippy(&self, args: &[String]) -> Result<()> { + self.command("rustup") + .args(["component", "add", "clippy"]) + .run()?; + self.script("ci/scripts/rust_clippy.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-check ...`. + // See the corresponding `rust-check` StepInfo for supported args. + pub(crate) fn run_rust_check(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("workspace"); + if args.len() > 1 { + return Err("usage: cargo xtask ci step rust-check [target]".to_string()); + } + + match target { + "workspace" => self + .command("cargo") + .args([ + "check", + "--profile", + "ci", + "--workspace", + "--all-targets", + "--features", + "integration-tests", + "--locked", + ]) + .run(), + "datafusion-common" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-common", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-common", + ]) + .run() + } + "datafusion-substrait" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-substrait", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + "--features=physical", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + "--features=protoc", + ]) + .run() + } + "datafusion-proto" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-proto", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-proto", + ]) + .run()?; + for feature in ["json", "parquet", "avro"] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-proto", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + "datafusion" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ]) + .run()?; + for feature in [ + "nested_expressions", + "array_expressions", + "avro", + "backtrace", + "compression", + "crypto_expressions", + "datetime_expressions", + "encoding_expressions", + "force_hash_collisions", + "math_expressions", + "parquet", + "regex_expressions", + "recursive_protection", + "serde", + "sql", + "string_expressions", + "unicode_expressions", + "parquet_encryption", + ] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + "datafusion-functions" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-functions", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-functions", + ]) + .run()?; + for feature in [ + "crypto_expressions", + "datetime_expressions", + "encoding_expressions", + "math_expressions", + "regex_expressions", + "string_expressions", + "unicode_expressions", + ] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-functions", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + unknown => Err(format!("unknown rust-check target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step rust-test ...`. + // See the corresponding `rust-test` StepInfo for supported args. + pub(crate) fn run_rust_test(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("workspace"); + if args.len() > 1 { + return Err( + "usage: cargo xtask ci step rust-test [workspace|datafusion-cli|datafusion-ffi]" + .to_string(), + ); + } + + match target { + "workspace" => self + .command("cargo") + .args([ + "test", + "--profile", + "ci", + "--exclude", + "datafusion-examples", + "--exclude", + "ffi_example_table_provider", + "--exclude", + "datafusion-cli", + "--workspace", + "--lib", + "--tests", + "--bins", + "--features", + "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait", + ]) + .env("RUST_BACKTRACE", "1") + .run(), + "datafusion-cli" => self + .command("cargo") + .args([ + "test", + "--features", + "backtrace", + "--profile", + "ci", + "-p", + "datafusion-cli", + "--lib", + "--tests", + "--bins", + ]) + .env("RUST_BACKTRACE", "1") + .env("AWS_ENDPOINT", "http://127.0.0.1:9000";) + .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin") + .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword") + .env("TEST_STORAGE_INTEGRATION", "1") + .env("AWS_ALLOW_HTTP", "true") + .run(), + "datafusion-ffi" => self + .command("cargo") + .args([ + "test", + "--profile", + "ci", + "-p", + "datafusion-ffi", + "--lib", + "--tests", + "--features", + "integration-tests", + ]) + .run(), + unknown => Err(format!("unknown rust-test target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step rust-doctest ...`. + // See the corresponding `rust-doctest` StepInfo for supported args. + pub(crate) fn run_rust_doctest(&self, _args: &[String]) -> Result<()> { + self.command("cargo") + .args([ + "test", + "--profile", + "ci", + "--doc", + "--features", + "avro,json", + ]) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-doc ...`. + // See the corresponding `rust-doc` StepInfo for supported args. + pub(crate) fn run_rust_doc(&self, _args: &[String]) -> Result<()> { + self.script("ci/scripts/rust_docs.sh").run() + } + + // Step runner entry point for `cargo xtask ci step rust-examples ...`. + // See the corresponding `rust-examples` StepInfo for supported args. + pub(crate) fn run_rust_examples(&self, _args: &[String]) -> Result<()> { + self.command("cargo") + .args(["run", "--profile", "ci", "--example", "sql"]) + .run()?; + self.script("ci/scripts/rust_example.sh").run() + } + + // Step runner entry point for `cargo xtask ci step rust-wasm ...`. + // See the corresponding `rust-wasm` StepInfo for supported args. + pub(crate) fn run_rust_wasm(&self, _args: &[String]) -> Result<()> { + let wasm_dir = self.root.join("datafusion/wasmtest"); + let rustflags = r#"--cfg getrandom_backend="wasm_js" -C debuginfo=none"#; + self.command("wasm-pack") + .args(["test", "--headless", "--firefox"]) + .current_dir(&wasm_dir) + .env("RUSTFLAGS", rustflags) + .run()?; + + let chrome_driver = format!( Review Comment: This results in wasm-pack being called with --chromedriver /chromedriver , which fails with a confusing "not found" error. The run_rust_sqllogictest function handles the missing-env-var case much better by returning an explicit error, we can use the same here ########## xtask/src/main.rs: ########## @@ -0,0 +1,361 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! See `cargo xtask help` for usage and `.github/workflows/rust.yml` for architecture. + +mod ci_jobs; +mod ci_timing; +mod ci_workflow_rust; +mod ci_workflows; +mod step_runners; +mod steps; +mod utils; + +use ci_jobs::JOBS as CI_JOBS; +use ci_timing::{JobTiming, WorkflowTiming, WorkflowsTiming}; +use ci_workflows::{WORKFLOWS as CI_WORKFLOWS, WorkflowInfo}; +use std::env; +use std::path::PathBuf; +use std::time::SystemTime; +use steps::STEPS; + +pub(crate) type Result<T> = std::result::Result<T, String>; + +fn main() { + let args = env::args().skip(1).collect::<Vec<_>>(); + if let Err(error) = Xtask::new().and_then(|xtask| xtask.run(&args)) { + eprintln!("error: {error}"); + std::process::exit(1); + } +} + +pub(crate) struct Xtask { + pub(crate) root: PathBuf, +} + +impl Xtask { + fn new() -> Result<Self> { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let root = manifest_dir + .parent() + .ok_or_else(|| { + format!( + "could not find workspace root from {}", + manifest_dir.display() + ) + })? + .to_path_buf(); + + Ok(Self { root }) + } + + fn run(&self, args: &[String]) -> Result<()> { + let Some((command, rest)) = args.split_first() else { + self.print_help(); + return Ok(()); + }; + + if matches!(command.as_str(), "-h" | "--help" | "help" | "list") { + self.print_help(); + return Ok(()); + } + + match command.as_str() { + "ci" => self.ci(rest), + unknown => Err(format!( + "unknown command `{unknown}`. Run `cargo xtask help`." + )), + } + } + + fn print_help(&self) { + println!("DataFusion local CI runner"); + println!(); + println!("Run the same CI checks locally that GitHub Actions schedules."); + println!(); + println!("Quick Start:"); + println!(" Syntax:"); + println!(" cargo xtask ci step <step> [args]"); + println!(" cargo xtask ci job <job>"); + println!(" cargo xtask ci workflow [name]"); + println!(); + println!(" Examples:"); + println!(" cargo xtask ci step rust-test"); + println!(" Run one atomic Rust test step."); + println!(" cargo xtask ci job linux-test"); + println!(" Run one GitHub Actions job locally."); + println!(" cargo xtask ci step fmt --write --allow-dirty"); + println!(" Run best-effort formatting auto-fixes."); + println!(" cargo xtask ci workflow rust"); + println!(" Reproduce `.github/workflows/rust.yml` locally."); + println!(); + println!("Concepts:"); + println!( + " step Atomic CI check item, runnable with `cargo xtask ci step <step>`" + ); + println!(" job Ordered list of steps matching one GitHub Actions job"); + println!(" workflow Ordered list of jobs matching a GitHub workflow file"); + println!( + " Example: `cargo xtask ci workflow rust` reproduces `.github/workflows/rust.yml`" + ); + println!(); + println!("Compatibility:"); + println!( + " cargo xtask ci run <step> is still accepted as an alias for `ci step`." + ); + println!(); + println!("Common flags:"); + println!( + " --write Perform best-effort auto-fixes for applicable lint errors (e.g. `cargo fmt`)" + ); + println!( + " --allow-dirty Allow `--write` to change local files with uncommitted changes" + ); + println!(); + self.print_commands(); + } + + fn print_commands(&self) { + println!("Steps:"); + let width = STEPS + .iter() + .map(|step| format!("ci step {}", step.help_usage).len()) + .max() + .unwrap_or_default(); + + for step in STEPS { + println!( + " {:<width$} {}", + format!("ci step {}", step.help_usage), + step.help_description + ); + } + println!(); + println!("Jobs:"); + + let width = CI_JOBS + .iter() + .map(|job| format!("ci job {}", job.name).len()) + .max() + .unwrap_or_default(); + + for job in CI_JOBS { + println!( + " {:<width$} {}", + format!("ci job {}", job.name), + job.help_description + ); + } + println!(); + println!("Workflows:"); + + let width = CI_WORKFLOWS + .iter() + .map(|workflow| workflow.help_usage.len()) + .chain(std::iter::once("ci workflow".len())) + .max() + .unwrap_or_default(); + + println!(" {:<width$} Run all CI workflows locally", "ci workflow"); + for workflow in CI_WORKFLOWS { + println!( + " {:<width$} {}", + workflow.help_usage, workflow.help_description + ); + } + println!(" (WIP: add remaining GitHub workflows)"); + } + + fn ci(&self, args: &[String]) -> Result<()> { + let Some((command, rest)) = args.split_first() else { + return Err("usage: cargo xtask ci <step|job|workflow> ...".to_string()); + }; + + match command.as_str() { + "step" | "run" => self.ci_step(rest), + "job" => self.ci_job(rest), + "workflow" => self.ci_workflow(rest), + "profile" => Err( + "cargo xtask ci profile is reserved for future local profiles" + .to_string(), + ), + "-h" | "--help" | "help" => { + self.print_help(); + Ok(()) + } + unknown => Err(format!( + "unknown ci command `{unknown}`. Run `cargo xtask help`." + )), + } + } + + fn ci_step(&self, args: &[String]) -> Result<()> { + let Some((command, rest)) = args.split_first() else { + return Err("usage: cargo xtask ci step <step> [args]".to_string()); + }; + + let step = steps::find_step(command).ok_or_else(|| { + format!("unknown command `{command}`. Run `cargo xtask help`.") + })?; + + let runner = step.runner; + let start = SystemTime::now(); + let result = runner(self, rest); + println!( + "\nci step `{command}` completed in {:?}", + start.elapsed().unwrap_or_default() + ); + + result.map_err(|error| { + format!( + "{}\n\nUnderlying command error:\n {error}", + step.error_message + ) + }) + } + + fn ci_job(&self, args: &[String]) -> Result<()> { + let [job_name] = args else { + return Err("usage: cargo xtask ci job <job>".to_string()); + }; + + let job = CI_JOBS + .iter() + .find(|job| job.name == job_name.as_str()) + .ok_or_else(|| "usage: cargo xtask ci job <job>".to_string())?; + + let mut job_timing = JobTiming::new(job); + for step_call in job.steps { + let step = step_call.step()?; + let args = step_call + .args + .iter() + .map(|arg| arg.to_string()) + .collect::<Vec<_>>(); + let runner = step.runner; + let start = SystemTime::now(); + let result = runner(self, &args); + job_timing.add_step_timing(step, start.elapsed().unwrap_or_default()); + + if let Err(error) = result Review Comment: we should consider building the full message in a single `map_err` rather than chaining, there are several instances -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
