This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ea7016f [SPARK-35411][SQL][FOLLOWUP] Handle Currying Product while
serializing TreeNode to JSON
ea7016f is described below
commit ea7016f6f9ab48b345211f53d121f15708245106
Author: Tengfei Huang <[email protected]>
AuthorDate: Mon May 31 22:15:26 2021 +0800
[SPARK-35411][SQL][FOLLOWUP] Handle Currying Product while serializing
TreeNode to JSON
### What changes were proposed in this pull request?
Handle Currying Product while serializing TreeNode to JSON. While
processing
[Product](https://github.com/apache/spark/blob/v3.1.2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala#L820),
we may get an assert error for cases like Currying Product because of the
mismatch of sizes between field name and field values.
Fallback to use reflection to get all the values for constructor parameters
when we meet such cases.
### Why are the changes needed?
Avoid throwing error while serializing TreeNode to JSON, try to output as
much information as possible.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
New UT case added.
Closes #32713 from ivoson/SPARK-35411-followup.
Authored-by: Tengfei Huang <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/catalyst/trees/TreeNode.scala | 16 +++++++++++++++-
.../apache/spark/sql/catalyst/trees/TreeNodeSuite.scala | 17 +++++++++++++++++
2 files changed, 32 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index d6da04e..8614eda 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -816,7 +816,20 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]]
extends Product {
case p: Product if shouldConvertToJson(p) =>
try {
val fieldNames = getConstructorParameterNames(p.getClass)
- val fieldValues = p.productIterator.toSeq
+ val fieldValues = {
+ if (p.productArity == fieldNames.length) {
+ p.productIterator.toSeq
+ } else {
+ val clazz = p.getClass
+ // Fallback to use reflection if length of product elements do not
match
+ // constructor params.
+ fieldNames.map { fieldName =>
+ val field = clazz.getDeclaredField(fieldName)
+ field.setAccessible(true)
+ field.get(p)
+ }
+ }
+ }
assert(fieldNames.length == fieldValues.length, s"$simpleClassName
fields: " +
fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(",
"))
("product-class" -> JString(p.getClass.getName)) ::
fieldNames.zip(fieldValues).map {
@@ -824,6 +837,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]]
extends Product {
}.toList
} catch {
case _: RuntimeException => null
+ case _: ReflectiveOperationException => null
}
case _ => JNull
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index d837af7..9afad22 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -88,6 +88,8 @@ case class FakeLeafPlan(child: LogicalPlan)
override def output: Seq[Attribute] = child.output
}
+case class FakeCurryingProduct(x: Expression)(val y: Int)
+
class TreeNodeSuite extends SparkFunSuite with SQLHelper {
test("top node changed") {
val after = Literal(1) transform { case Literal(1, _) => Literal(2) }
@@ -615,6 +617,21 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
"num-children" -> 0,
"arg" -> "1"
)))))
+
+ // Convert currying product contains TreeNode to JSON.
+ assertJSON(
+ FakeCurryingProduct(Literal(1))(1),
+ JObject(
+ "product-class" -> classOf[FakeCurryingProduct].getName,
+ "x" -> List(
+ JObject(
+ "class" -> JString(classOf[Literal].getName),
+ "num-children" -> 0,
+ "value" -> "1",
+ "dataType" -> "integer")),
+ "y" -> 1
+ )
+ )
}
test("toJSON should not throws java.lang.StackOverflowError") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
