This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 1b8117eae add session option 'datafusion.explain.logical_plan'. when
set to true, the explain statement will only print logical plans. (#2895)
1b8117eae is described below
commit 1b8117eaeca586e40882a80a53103f1be17d2b1c
Author: AssHero <[email protected]>
AuthorDate: Thu Jul 14 18:32:05 2022 +0800
add session option 'datafusion.explain.logical_plan'. when set to true, the
explain statement will only print logical plans. (#2895)
* add session option 'datafusion.explain.logical_plan'. when set to true,
the explain statement will only print logical plans.
* 1. add test cases 2. add session options
'datafusion.explain.logical_plan_only' and
'datafusion.explain.physical_plan_only' to
seperately print logical plans and physical plans.
* update docs/source/user-guide/configs.md
---
datafusion/core/src/config.rs | 16 ++++++++++
datafusion/core/src/physical_plan/planner.rs | 47 +++++++++++++++++++---------
datafusion/core/tests/sql/explain_analyze.rs | 41 +++++++++++++++++++++++-
docs/source/user-guide/configs.md | 2 ++
4 files changed, 90 insertions(+), 16 deletions(-)
diff --git a/datafusion/core/src/config.rs b/datafusion/core/src/config.rs
index 20aa0055b..f732d6431 100644
--- a/datafusion/core/src/config.rs
+++ b/datafusion/core/src/config.rs
@@ -27,6 +27,12 @@ use std::env;
/// Configuration option "datafusion.optimizer.filter_null_join_keys"
pub const OPT_FILTER_NULL_JOIN_KEYS: &str =
"datafusion.optimizer.filter_null_join_keys";
+/// Configuration option "datafusion.explain.logical_plan_only"
+pub const OPT_EXPLAIN_LOGICAL_PLAN_ONLY: &str =
"datafusion.explain.logical_plan_only";
+
+/// Configuration option "datafusion.explain.physical_plan_only"
+pub const OPT_EXPLAIN_PHYSICAL_PLAN_ONLY: &str =
"datafusion.explain.physical_plan_only";
+
/// Configuration option "datafusion.execution.batch_size"
pub const OPT_BATCH_SIZE: &str = "datafusion.execution.batch_size";
@@ -119,6 +125,16 @@ impl BuiltInConfigs {
predicate push down.",
false,
),
+ ConfigDefinition::new_bool(
+ OPT_EXPLAIN_LOGICAL_PLAN_ONLY,
+ "When set to true, the explain statement will only print
logical plans.",
+ false,
+ ),
+ ConfigDefinition::new_bool(
+ OPT_EXPLAIN_PHYSICAL_PLAN_ONLY,
+ "When set to true, the explain statement will only print
physical plans.",
+ false,
+ ),
ConfigDefinition::new_u64(
OPT_BATCH_SIZE,
"Default batch size while creating new batches, it's especially
useful for \
diff --git a/datafusion/core/src/physical_plan/planner.rs
b/datafusion/core/src/physical_plan/planner.rs
index 40daef606..820808a08 100644
--- a/datafusion/core/src/physical_plan/planner.rs
+++ b/datafusion/core/src/physical_plan/planner.rs
@@ -22,6 +22,7 @@ use super::{
aggregates, empty::EmptyExec, hash_join::PartitionMode, udaf,
union::UnionExec,
values::ValuesExec, windows,
};
+use crate::config::{OPT_EXPLAIN_LOGICAL_PLAN_ONLY,
OPT_EXPLAIN_PHYSICAL_PLAN_ONLY};
use crate::datasource::source_as_provider;
use crate::execution::context::{ExecutionProps, SessionState};
use crate::logical_expr::utils::generate_sort_key;
@@ -1487,26 +1488,42 @@ impl DefaultPhysicalPlanner {
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
if let LogicalPlan::Explain(e) = logical_plan {
use PlanType::*;
- let mut stringified_plans = e.stringified_plans.clone();
+ let mut stringified_plans = vec![];
- stringified_plans.push(e.plan.to_stringified(FinalLogicalPlan));
+ if !session_state
+ .config
+ .config_options
+ .get_bool(OPT_EXPLAIN_PHYSICAL_PLAN_ONLY)
+ {
+ stringified_plans = e.stringified_plans.clone();
- let input = self
- .create_initial_plan(e.plan.as_ref(), session_state)
- .await?;
+
stringified_plans.push(e.plan.to_stringified(FinalLogicalPlan));
+ }
+
+ if !session_state
+ .config
+ .config_options
+ .get_bool(OPT_EXPLAIN_LOGICAL_PLAN_ONLY)
+ {
+ let input = self
+ .create_initial_plan(e.plan.as_ref(), session_state)
+ .await?;
- stringified_plans
-
.push(displayable(input.as_ref()).to_stringified(InitialPhysicalPlan));
+ stringified_plans.push(
+
displayable(input.as_ref()).to_stringified(InitialPhysicalPlan),
+ );
- let input =
- self.optimize_internal(input, session_state, |plan, optimizer|
{
- let optimizer_name = optimizer.name().to_string();
- let plan_type = OptimizedPhysicalPlan { optimizer_name };
-
stringified_plans.push(displayable(plan).to_stringified(plan_type));
- })?;
+ let input =
+ self.optimize_internal(input, session_state, |plan,
optimizer| {
+ let optimizer_name = optimizer.name().to_string();
+ let plan_type = OptimizedPhysicalPlan { optimizer_name
};
+ stringified_plans
+ .push(displayable(plan).to_stringified(plan_type));
+ })?;
- stringified_plans
-
.push(displayable(input.as_ref()).to_stringified(FinalPhysicalPlan));
+ stringified_plans
+
.push(displayable(input.as_ref()).to_stringified(FinalPhysicalPlan));
+ }
Ok(Some(Arc::new(ExplainExec::new(
SchemaRef::new(e.schema.as_ref().to_owned().into()),
diff --git a/datafusion/core/tests/sql/explain_analyze.rs
b/datafusion/core/tests/sql/explain_analyze.rs
index 1ca0a41d4..02db3e873 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -16,7 +16,10 @@
// under the License.
use super::*;
-use datafusion::physical_plan::display::DisplayableExecutionPlan;
+use datafusion::{
+ config::{OPT_EXPLAIN_LOGICAL_PLAN_ONLY, OPT_EXPLAIN_PHYSICAL_PLAN_ONLY},
+ physical_plan::display::DisplayableExecutionPlan,
+};
#[tokio::test]
async fn explain_analyze_baseline_metrics() {
@@ -810,3 +813,39 @@ async fn csv_explain_analyze_verbose() {
let verbose_needle = "Output Rows";
assert_contains!(formatted, verbose_needle);
}
+
+#[tokio::test]
+async fn explain_logical_plan_only() {
+ let config = SessionConfig::new().set_bool(OPT_EXPLAIN_LOGICAL_PLAN_ONLY,
true);
+ let ctx = SessionContext::with_config(config);
+ let sql = "EXPLAIN select count(*) from (values ('a', 1, 100), ('a', 2,
150)) as t (c1,c2,c3)";
+ let actual = execute(&ctx, sql).await;
+ let actual = normalize_vec_for_explain(actual);
+
+ let expected = vec![
+ vec![
+ "logical_plan",
+ "Projection: #COUNT(UInt8(1))\
+ \n Aggregate: groupBy=[[]], aggr=[[COUNT(UInt8(1))]]\
+ \n Values: (Utf8(\"a\"), Int64(1), Int64(100)), (Utf8(\"a\"),
Int64(2), Int64(150))",
+ ]];
+ assert_eq!(expected, actual);
+}
+
+#[tokio::test]
+async fn explain_physical_plan_only() {
+ let config = SessionConfig::new().set_bool(OPT_EXPLAIN_PHYSICAL_PLAN_ONLY,
true);
+ let ctx = SessionContext::with_config(config);
+ let sql = "EXPLAIN select count(*) from (values ('a', 1, 100), ('a', 2,
150)) as t (c1,c2,c3)";
+ let actual = execute(&ctx, sql).await;
+ let actual = normalize_vec_for_explain(actual);
+
+ let expected = vec![vec![
+ "physical_plan",
+ "ProjectionExec: expr=[COUNT(UInt8(1))@0 as COUNT(UInt8(1))]\
+ \n ProjectionExec: expr=[2 as COUNT(UInt8(1))]\
+ \n EmptyExec: produce_one_row=true\
+ \n",
+ ]];
+ assert_eq!(expected, actual);
+}
diff --git a/docs/source/user-guide/configs.md
b/docs/source/user-guide/configs.md
index c899bafdb..41f6df850 100644
--- a/docs/source/user-guide/configs.md
+++ b/docs/source/user-guide/configs.md
@@ -40,4 +40,6 @@ Environment variables are read during `SessionConfig`
initialisation so they mus
| datafusion.execution.batch_size | UInt64 | 8192 |
Default batch size while creating new batches, it's especially useful for
buffer-in-memory batches since creating tiny batches would results in too much
metadata memory consumption.
|
| datafusion.execution.coalesce_batches | Boolean | true | When
set to true, record batches will be examined between each operator and small
batches will be coalesced into larger batches. This is helpful when there are
highly selective filters or joins that could produce tiny output batches. The
target batch size is determined by the configuration setting
'datafusion.execution.coalesce_target_batch_size'. |
| datafusion.execution.coalesce_target_batch_size | UInt64 | 4096 | Target
batch size when coalescing batches. Uses in conjunction with the configuration
setting 'datafusion.execution.coalesce_batches'.
|
+| datafusion.explain.logical_plan_only | Boolean | false | When
set to true, the explain statement will only print logical plans.
|
+| datafusion.explain.physical_plan_only | Boolean | false | When
set to true, the explain statement will only print physical plans.
|
| datafusion.optimizer.filter_null_join_keys | Boolean | false | When
set to true, the optimizer will insert filters before a join between a nullable
and non-nullable column to filter out nulls on the nullable side. This filter
can add additional overhead when the file format does not fully support
predicate push down.
|
