This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 326b7df3c ci: fix lint scripts that skipped checks or passed without
running (#2473)
326b7df3c is described below
commit 326b7df3c7443d6ee381b3d665eff0cd91bd7428
Author: NoahKusaba <[email protected]>
AuthorDate: Wed Sep 23 18:24:26 2026 -0400
ci: fix lint scripts that skipped checks or passed without running (#2473)
* ci: fix lint scripts that skipped checks or passed without running
- run-rat.sh invoked `python`, not `python3`; with no `python` on PATH the
checker never ran and the script reported no unapproved licenses. It now
fails when the checker does not run.
- rust_clippy.sh named crates explicitly, leaving ballista-api-types,
ballista-history and ballista-chaos unlinted. Use --workspace, keeping
ballista-cli separate because its tui and web features can't be combined.
- dev/rust_lint.sh ran four of the seven CI lint scripts.
- pre-commit.sh required an uninstalled nightly toolchain for cargo fmt.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* ci: fail run-rat.sh whenever the licence check did not actually run
python3 exits 1 on an uncaught exception, the same status the checker uses
for unapproved files, so an empty or malformed rat.txt still reported no
unapproved licenses. Decide from the checker's exit status together with
its NOT APPROVED lines instead: success needs exit 0, and a failure with no
NOT APPROVED lines is reported as the checker failing.
Also fail when RAT itself fails, and download the jar with curl -f so an
HTTP error is not cached as the jar and reused on later runs.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
ci/scripts/rust_clippy.sh | 8 ++------
dev/release/run-rat.sh | 27 +++++++++++++++++++++------
dev/rust_lint.sh | 4 ++++
pre-commit.sh | 22 +++++++++-------------
4 files changed, 36 insertions(+), 25 deletions(-)
diff --git a/ci/scripts/rust_clippy.sh b/ci/scripts/rust_clippy.sh
index e0f72ce17..acbb6df22 100755
--- a/ci/scripts/rust_clippy.sh
+++ b/ci/scripts/rust_clippy.sh
@@ -18,11 +18,7 @@
# under the License.
set -ex
-cargo clippy --all-targets --package ballista-core --all-features -- -D
warnings
-cargo clippy --all-targets --package ballista-scheduler --all-features -- -D
warnings
-cargo clippy --all-targets --package ballista-executor --all-features -- -D
warnings
-cargo clippy --all-targets --package ballista --all-features -- -D warnings
+# ballista-cli is linted separately: its tui and web features can't be
combined.
+cargo clippy --all-targets --workspace --exclude ballista-cli --all-features
-- -D warnings
cargo clippy --all-targets --package ballista-cli --no-default-features
--features cli,tui -- -D warnings
cargo clippy --all-targets --package ballista-cli --no-default-features
--features web -- -D warnings
-cargo clippy --all-targets --package ballista-examples --all-features -- -D
warnings
-cargo clippy --all-targets --package ballista-benchmarks --all-features -- -D
warnings
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index 94fa55fbe..50fb5a345 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -22,7 +22,12 @@ RAT_VERSION=0.13
# download apache rat
if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then
- curl -s
https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar
> apache-rat-${RAT_VERSION}.jar
+ # -f so an HTTP error fails here instead of caching an error page as the jar.
+ if ! curl -sSfL -o apache-rat-${RAT_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar;
then
+ rm -f apache-rat-${RAT_VERSION}.jar
+ echo "Failed to download apache-rat-${RAT_VERSION}.jar"
+ exit 1
+ fi
fi
RAT="java -jar apache-rat-${RAT_VERSION}.jar -x "
@@ -30,14 +35,24 @@ RAT="java -jar apache-rat-${RAT_VERSION}.jar -x "
RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
# generate the rat report
-$RAT $1 > rat.txt
-python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt
rat.txt > filtered_rat.txt
+if ! $RAT $1 > rat.txt; then
+ echo "Apache RAT failed; see the error above"
+ exit 1
+fi
+
+python3 $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt
rat.txt > filtered_rat.txt
+CHECK_STATUS=$?
cat filtered_rat.txt
-UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l`
+UNAPPROVED=`grep -c "NOT APPROVED" filtered_rat.txt`
-if [ "0" -eq "${UNAPPROVED}" ]; then
+# A nonzero exit with no NOT APPROVED lines means the checker itself failed,
+# e.g. python3 missing or rat.txt not being a valid report.
+if [ "${CHECK_STATUS}" -eq 0 ]; then
echo "No unapproved licenses"
-else
+elif [ "${UNAPPROVED}" -gt 0 ]; then
echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
exit 1
+else
+ echo "check-rat-report.py failed (exit ${CHECK_STATUS}); see rat.txt"
+ exit 1
fi
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index 6e680dbb9..e88aa16f3 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -18,7 +18,11 @@
# under the License.
set -e
+# Keep in sync with the lint jobs in .github/workflows/rust.yml and dev.yml.
ci/scripts/rust_fmt.sh
ci/scripts/rust_clippy.sh
ci/scripts/rust_toml_fmt.sh
ci/scripts/rust_machete.sh
+ci/scripts/rust_config_docs_check.sh
+ci/scripts/rust_proto_check.sh
+ci/scripts/rust_docs.sh
diff --git a/pre-commit.sh b/pre-commit.sh
index f82390e22..26deb3a7e 100755
--- a/pre-commit.sh
+++ b/pre-commit.sh
@@ -19,7 +19,7 @@
# This file is git pre-commit hook.
#
-# Soft link it as git hook under top dir of apache arrow git repository:
+# Soft link it as a git hook from the top level of this repository:
# $ ln -s ../../pre-commit.sh .git/hooks/pre-commit
#
# This file be run directly:
@@ -63,20 +63,16 @@ echo -e "$(GREEN INFO): cargo clippy ..."
cargo clippy
echo -e "$(GREEN INFO): cargo clippy done"
-# 2. cargo fmt: format with nightly and stable.
+# 2. cargo fmt: stable only, matching rust-toolchain.toml and CI.
CHANGED_BY_CARGO_FMT=false
-echo -e "$(GREEN INFO): cargo fmt with nightly and stable ..."
-
-for version in nightly stable; do
- CMD="cargo +${version} fmt"
- ${CMD} --all -q -- --check 2>/dev/null
- if [ $? -ne 0 ]; then
- ${CMD} --all
- echo -e "$(BYELLOW WARN): ${CMD} changed some files"
- CHANGED_BY_CARGO_FMT=true
- fi
-done
+echo -e "$(GREEN INFO): cargo fmt ..."
+
+if ! cargo fmt --all -q -- --check 2>/dev/null; then
+ cargo fmt --all
+ echo -e "$(BYELLOW WARN): cargo fmt changed some files"
+ CHANGED_BY_CARGO_FMT=true
+fi
if ${CHANGED_BY_CARGO_FMT}; then
echo -e "$(RED FAIL): git commit $(RED ABORTED), please have a look and
run git add/commit again"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
