This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 60c5b26c9 chore(scheduler,executor): downgrade #2038 per-task traces
to debug (#2104)
60c5b26c9 is described below
commit 60c5b26c914d30c0b1265862bc9e399b54bb72ec
Author: Brent Gardner <[email protected]>
AuthorDate: Mon Jul 20 09:35:27 2026 -0600
chore(scheduler,executor): downgrade #2038 per-task traces to debug (#2104)
Follow-up to #2038 (multi-partition tasks). Four log sites introduced
by that PR fire on every scheduling/execution decision and flood
scheduler + executor logs at INFO:
* `ballista/scheduler/src/cluster/mod.rs`:
`stage_has_input_collapse` + `bind_one` — per-bind decision traces
(previously prefixed with an ad-hoc `DIAG ` tag, dropped here).
* `ballista/executor/src/execution_engine.rs`:
`executor plan pre-run` + `executor plan post-run` — per-task
DisplayableExecutionPlan dumps. Pre-#2038 this function had no
logging; both dumps are noise for anyone not actively debugging.
All four move to `debug!`. Turn them back on with
`RUST_LOG=ballista_scheduler=debug,ballista_executor=debug`.
Closes #2103.
Co-authored-by: Claude Opus 4.7 (1M context) <[email protected]>
---
ballista/executor/src/execution_engine.rs | 6 +++---
ballista/scheduler/src/cluster/mod.rs | 10 +++++-----
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/ballista/executor/src/execution_engine.rs
b/ballista/executor/src/execution_engine.rs
index 5633a6dde..2a83e762b 100644
--- a/ballista/executor/src/execution_engine.rs
+++ b/ballista/executor/src/execution_engine.rs
@@ -38,7 +38,7 @@ use
datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::physical_plan::metrics::MetricsSet;
use datafusion::prelude::SessionConfig;
use futures::stream::TryStreamExt;
-use log::info;
+use log::debug;
use std::fmt::{Debug, Display};
use std::sync::Arc;
@@ -260,7 +260,7 @@ impl QueryStageExecutor for DefaultQueryStageExec {
ShuffleWriterVariant::Hash(writer) =>
(Arc::new(writer.clone()), false),
ShuffleWriterVariant::Sort(writer) =>
(Arc::new(writer.clone()), true),
};
- info!(
+ debug!(
"executor plan pre-run (task_id={task_id}):\n{}",
DisplayableExecutionPlan::new(plan_arc.as_ref()).indent(true)
);
@@ -272,7 +272,7 @@ impl QueryStageExecutor for DefaultQueryStageExec {
let result =
drive_shuffle_writer_stage(plan_arc.clone(), context,
is_sort_shuffle).await;
- info!(
+ debug!(
"executor plan post-run (task_id={task_id}, ok={}):\n{}",
result.is_ok(),
DisplayableExecutionPlan::with_metrics(plan_arc.as_ref()).indent(true)
diff --git a/ballista/scheduler/src/cluster/mod.rs
b/ballista/scheduler/src/cluster/mod.rs
index cea3858d1..43de62753 100644
--- a/ballista/scheduler/src/cluster/mod.rs
+++ b/ballista/scheduler/src/cluster/mod.rs
@@ -34,7 +34,7 @@ use ballista_core::{ConfigProducer, JobId,
JobStatusSubscriber};
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::{SessionConfig, SessionContext};
use futures::Stream;
-use log::{debug, info};
+use log::debug;
use std::collections::{HashMap, HashSet};
use std::pin::Pin;
use std::sync::Arc;
@@ -392,8 +392,8 @@ fn stage_has_input_collapse(plan_root: &Arc<dyn
ExecutionPlan>) -> bool {
[child] => walk(child),
_ => false,
};
- info!(
- "DIAG stage_has_input_collapse: root={} root_partitions={} → {result}",
+ debug!(
+ "stage_has_input_collapse: root={} root_partitions={} → {result}",
plan_root.name(),
plan_root
.properties()
@@ -461,8 +461,8 @@ fn bind_one(
} else {
input_partition_ids.len() as u32
};
- info!(
- "DIAG bind_one: job={} stage={} exec={} vcores={} max={} slice_len={}
consumed={} partitions={:?} collapse={}",
+ debug!(
+ "bind_one: job={} stage={} exec={} vcores={} max={} slice_len={}
consumed={} partitions={:?} collapse={}",
job_id,
running_stage.stage_id,
budget.executor_id,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
