This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 0f13606bb55 [SPARK-35378][SQL][FOLLOW-UP] Fix incorrect return type in
CommandResultExec.executeCollect()
0f13606bb55 is described below
commit 0f13606bb55087da657b87d0c2f5a5583ed75e6c
Author: Ivan Sadikov <[email protected]>
AuthorDate: Mon May 23 16:58:36 2022 +0800
[SPARK-35378][SQL][FOLLOW-UP] Fix incorrect return type in
CommandResultExec.executeCollect()
### What changes were proposed in this pull request?
This PR is a follow-up for https://github.com/apache/spark/pull/32513 and
fixes an issue introduced by that patch.
CommandResultExec is supposed to return `UnsafeRow` records in all of the
`executeXYZ` methods but `executeCollect` was left out which causes issues like
this one:
```
Error in SQL statement: ClassCastException:
org.apache.spark.sql.catalyst.expressions.GenericInternalRow
cannot be cast to org.apache.spark.sql.catalyst.expressions.UnsafeRow
```
We need to return `unsafeRows` instead of `rows` in `executeCollect`
similar to other methods in the class.
### Why are the changes needed?
Fixes a bug in CommandResultExec.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
I added a unit test to check the return type of all commands.
Closes #36632 from sadikovi/fix-command-exec.
Authored-by: Ivan Sadikov <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit a0decfc7db68c464e3ba2c2fb0b79a8b0c464684)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../scala/org/apache/spark/sql/execution/CommandResultExec.scala | 4 ++--
.../org/apache/spark/sql/execution/QueryExecutionSuite.scala | 9 +++++++++
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/CommandResultExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/CommandResultExec.scala
index 37c8de983f4..21d1c97db98 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/CommandResultExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/CommandResultExec.scala
@@ -76,8 +76,8 @@ case class CommandResultExec(
}
override def executeCollect(): Array[InternalRow] = {
- longMetric("numOutputRows").add(rows.size)
- rows.toArray
+ longMetric("numOutputRows").add(unsafeRows.size)
+ unsafeRows
}
override def executeTake(limit: Int): Array[InternalRow] = {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
index 2c58b53969b..41a1cd9b294 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
@@ -20,6 +20,7 @@ import scala.io.Source
import org.apache.spark.sql.{AnalysisException, FastOperator}
import org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace
+import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical.{CommandResult,
LogicalPlan, OneRowRelation, Project, ShowTables, SubqueryAlias}
import org.apache.spark.sql.catalyst.trees.TreeNodeTag
@@ -262,6 +263,14 @@ class QueryExecutionSuite extends SharedSparkSession {
assert(cmdResultExec.commandPhysicalPlan.isInstanceOf[ShowTablesExec])
}
+ test("SPARK-35378: Return UnsafeRow in CommandResultExecCheck execute
methods") {
+ val plan = spark.sql("SHOW FUNCTIONS").queryExecution.executedPlan
+ assert(plan.isInstanceOf[CommandResultExec])
+ plan.executeCollect().foreach { row => assert(row.isInstanceOf[UnsafeRow])
}
+ plan.executeTake(10).foreach { row => assert(row.isInstanceOf[UnsafeRow]) }
+ plan.executeTail(10).foreach { row => assert(row.isInstanceOf[UnsafeRow]) }
+ }
+
test("SPARK-38198: check specify maxFields when call toFile method") {
withTempDir { dir =>
val path = dir.getCanonicalPath + "/plans.txt"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
