This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new ff92e85 [SPARK-38198][SQL] Fix `QueryExecution.debug#toFile` use the passed in `maxFields` when `explainMode` is `CodegenMode` ff92e85 is described below commit ff92e85f86d3e36428996695001a23893d406b76 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Mon Feb 14 13:28:11 2022 +0300 [SPARK-38198][SQL] Fix `QueryExecution.debug#toFile` use the passed in `maxFields` when `explainMode` is `CodegenMode` ### What changes were proposed in this pull request? `QueryExecution.debug#toFile` method supports passing in `maxFields` and this parameter will be passed down when `explainMode` is `SimpleMode`, `ExtendedMode`, or `CostMode`. But the passed down `maxFields` was ignored when `explainMode` is `CostMode` because `QueryExecution#stringWithStats` overrides it with `SQLConf.get.maxToStringFields` at present, so this pr removes the override behavior to let passed in `maxFields` take effect. ### Why are the changes needed? Bug fix ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GA and add a new test case Closes #35506 from LuciferYang/SPARK-38198. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../apache/spark/sql/execution/QueryExecution.scala | 2 -- .../spark/sql/execution/QueryExecutionSuite.scala | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala index 26c6904..1b08994 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala @@ -304,8 +304,6 @@ class QueryExecution( } private def stringWithStats(maxFields: Int, append: String => Unit): Unit = { - val maxFields = SQLConf.get.maxToStringFields - // trigger to compute stats for logical plans try { // This will trigger to compute stats for all the nodes in the plan, including subqueries, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala index ecc448f..2c58b53 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala @@ -261,4 +261,22 @@ class QueryExecutionSuite extends SharedSparkSession { val cmdResultExec = projectQe.executedPlan.asInstanceOf[CommandResultExec] assert(cmdResultExec.commandPhysicalPlan.isInstanceOf[ShowTablesExec]) } + + test("SPARK-38198: check specify maxFields when call toFile method") { + withTempDir { dir => + val path = dir.getCanonicalPath + "/plans.txt" + // Define a dataset with 6 columns + val ds = spark.createDataset(Seq((0, 1, 2, 3, 4, 5), (6, 7, 8, 9, 10, 11))) + // `CodegenMode` and `FormattedMode` doesn't use the maxFields, so not tested in this case + Seq(SimpleMode.name, ExtendedMode.name, CostMode.name).foreach { modeName => + val maxFields = 3 + ds.queryExecution.debug.toFile(path, explainMode = Some(modeName), maxFields = maxFields) + Utils.tryWithResource(Source.fromFile(path)) { source => + val tableScan = source.getLines().filter(_.contains("LocalTableScan")) + assert(tableScan.exists(_.contains("more fields")), + s"Specify maxFields = $maxFields doesn't take effect when explainMode is $modeName") + } + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org