Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/20753#discussion_r177468352
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
---
@@ -188,8 +189,30 @@ case class StaticInvoke(
override def nullable: Boolean = needNullCheck || returnNullable
override def children: Seq[Expression] = arguments
- override def eval(input: InternalRow): Any =
- throw new UnsupportedOperationException("Only code-generated
evaluation is supported.")
+ override def eval(input: InternalRow): Any = {
+ val args = arguments.map(e => e.eval(input).asInstanceOf[Object])
+ val argClasses = ScalaReflection.expressionJavaClasses(arguments)
+ val cls = if (staticObject.getName == objectName) {
+ staticObject
+ } else {
+ Utils.classForName(objectName)
+ }
+ val method = cls.getDeclaredMethod(functionName, argClasses : _*)
--- End diff --
Naive static resolution caused the following Serialization exception. Now,
I succeeded to resolve once using `lazy` as follows:
```
lazy val method = {
val argClasses = ScalaReflection.expressionJavaClasses(arguments)
val cls = if (staticObject.getName == objectName) {
staticObject
} else {
Utils.classForName(objectName)
}
cls.getDeclaredMethod(functionName, argClasses : _*)
}
```
```
java.lang.reflect.Method
Serialization stack:
- object not serializable (class: java.lang.reflect.Method, value:
public static java.lang.Boolean java.lang.Boolean.valueOf(java.lang.String))
- field (class:
org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke, name: method,
type: class java.lang.reflect.Method)
- object (class
org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke,
staticinvoke(class java.lang.Boolean, ObjectType(class java.lang.Boolean),
valueOf, input[0, java.lang.String, true], true, true))
java.io.NotSerializableException: java.lang.reflect.Method
Serialization stack:
- object not serializable (class: java.lang.reflect.Method, value:
public static java.lang.Boolean java.lang.Boolean.valueOf(java.lang.String))
- field (class:
org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke, name: method,
type: class java.lang.reflect.Method)
- object (class
org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke,
staticinvoke(class java.lang.Boolean, ObjectType(class java.lang.Boolean),
valueOf, input[0, java.lang.String, true], true, true))
at
org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
at
org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
at
org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
at
org.apache.spark.sql.catalyst.expressions.ObjectExpressionsSuite.org$apache$spark$sql$catalyst$expressions$ObjectExpressionsSuite$$checkObjectExprEvaluation(ObjectExpressionsSuite.scala:180)
at
org.apache.spark.sql.catalyst.expressions.ObjectExpressionsSuite$$anonfun$3$$anonfun$apply$mcV$sp$1.apply(ObjectExpressionsSuite.scala:99)
at
org.apache.spark.sql.catalyst.expressions.ObjectExpressionsSuite$$anonfun$3$$anonfun$apply$mcV$sp$1.apply(ObjectExpressionsSuite.scala:98)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.ObjectExpressionsSuite$$anonfun$3.apply$mcV$sp(ObjectExpressionsSuite.scala:98)
at
org.apache.spark.sql.catalyst.expressions.ObjectExpressionsSuite$$anonfun$3.apply(ObjectExpressionsSuite.scala:90)
at
org.apache.spark.sql.catalyst.expressions.ObjectExpressionsSuite$$anonfun$3.apply(ObjectExpressionsSuite.scala:90)
...
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
