This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ff92e85 [SPARK-38198][SQL] Fix `QueryExecution.debug#toFile` use the
passed in `maxFields` when `explainMode` is `CodegenMode`
ff92e85 is described below
commit ff92e85f86d3e36428996695001a23893d406b76
Author: yangjie01 <yangji...@baidu.com>
AuthorDate: Mon Feb 14 13:28:11 2022 +0300
[SPARK-38198][SQL] Fix `QueryExecution.debug#toFile` use the passed in
`maxFields` when `explainMode` is `CodegenMode`
### What changes were proposed in this pull request?
`QueryExecution.debug#toFile` method supports passing in `maxFields` and
this parameter will be passed down when `explainMode` is `SimpleMode`,
`ExtendedMode`, or `CostMode`.
But the passed down `maxFields` was ignored when `explainMode` is
`CostMode` because `QueryExecution#stringWithStats` overrides it with
`SQLConf.get.maxToStringFields` at present, so this pr removes the override
behavior to let passed in `maxFields` take effect.
### Why are the changes needed?
Bug fix
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GA and add a new test case
Closes #35506 from LuciferYang/SPARK-38198.
Authored-by: yangjie01 <yangji...@baidu.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../apache/spark/sql/execution/QueryExecution.scala | 2 --
.../spark/sql/execution/QueryExecutionSuite.scala | 18 ++++++++++++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 26c6904..1b08994 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -304,8 +304,6 @@ class QueryExecution(
}
private def stringWithStats(maxFields: Int, append: String => Unit): Unit = {
- val maxFields = SQLConf.get.maxToStringFields
-
// trigger to compute stats for logical plans
try {
// This will trigger to compute stats for all the nodes in the plan,
including subqueries,
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
index ecc448f..2c58b53 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
@@ -261,4 +261,22 @@ class QueryExecutionSuite extends SharedSparkSession {
val cmdResultExec = projectQe.executedPlan.asInstanceOf[CommandResultExec]
assert(cmdResultExec.commandPhysicalPlan.isInstanceOf[ShowTablesExec])
}
+
+ test("SPARK-38198: check specify maxFields when call toFile method") {
+ withTempDir { dir =>
+ val path = dir.getCanonicalPath + "/plans.txt"
+ // Define a dataset with 6 columns
+ val ds = spark.createDataset(Seq((0, 1, 2, 3, 4, 5), (6, 7, 8, 9, 10,
11)))
+ // `CodegenMode` and `FormattedMode` doesn't use the maxFields, so not
tested in this case
+ Seq(SimpleMode.name, ExtendedMode.name, CostMode.name).foreach {
modeName =>
+ val maxFields = 3
+ ds.queryExecution.debug.toFile(path, explainMode = Some(modeName),
maxFields = maxFields)
+ Utils.tryWithResource(Source.fromFile(path)) { source =>
+ val tableScan =
source.getLines().filter(_.contains("LocalTableScan"))
+ assert(tableScan.exists(_.contains("more fields")),
+ s"Specify maxFields = $maxFields doesn't take effect when
explainMode is $modeName")
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
