cfmcgrady commented on code in PR #4797:
URL: https://github.com/apache/kyuubi/pull/4797#discussion_r1189839665
##########
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala:
##########
@@ -312,6 +316,34 @@ object KyuubiArrowConverters extends SQLConfHelper with
Logging {
}
}
+ // the signature of function [[ArrowUtils.toArrowSchema]] is changed in
SPARK-41971 (since Spark
+ // 3.5)
+ private lazy val toArrowSchemaMethod = DynMethods.builder("toArrowSchema")
+ .impl( // for Spark 3.4 or previous
+ "org.apache.spark.sql.util.ArrowUtils",
+ classOf[StructType],
+ classOf[String])
+ .impl( // for Spark 3.5 or later
+ "org.apache.spark.sql.util.ArrowUtils",
+ classOf[StructType],
+ classOf[String],
+ classOf[Boolean])
+ .build()
+
+ /**
+ * this function uses reflective calls to the [[ArrowUtils.toArrowSchema]].
+ */
+ private def toArrowSchema(
+ schema: StructType,
+ timeZone: String,
+ errorOnDuplicatedFieldNames: Boolean): ArrowSchema = {
Review Comment:
FYI:
We ultimately decided to use autoboxing in our code since both methods
instantiate a `java.lang.Boolean` object.
```
// the bytecode of the method:
// ```
// private def toArrowSchema(
// schema: StructType,
// timeZone: String,
// errorOnDuplicatedFieldNames: Boolean): ArrowSchema = {
// toArrowSchemaMethod.invoke[ArrowSchema](
// ArrowUtils,
// schema,
// timeZone,
// errorOnDuplicatedFieldNames.asInstanceOf[Object])
// }
// ```
public org.apache.arrow.vector.types.pojo.Schema
org$apache$spark$sql$execution$arrow$KyuubiArrowConverters$$toArrowSchema(org.apache.spark.sql.types.StructType,
java.lang.String, boolean);
descriptor:
(Lorg/apache/spark/sql/types/StructType;Ljava/lang/String;Z)Lorg/apache/arrow/vector/types/pojo/Schema;
flags: ACC_PUBLIC
Code:
stack=6, locals=4, args_size=4
0: aload_0
1: invokespecial #592 // Method
toArrowSchemaMethod:()Lorg/apache/kyuubi/reflection/DynMethods$UnboundMethod;
4: getstatic #166 // Field
org/apache/spark/sql/util/ArrowUtils$.MODULE$:Lorg/apache/spark/sql/util/ArrowUtils$;
7: iconst_3
8: anewarray #4 // class java/lang/Object
11: dup
12: iconst_0
13: aload_1
14: aastore
15: dup
16: iconst_1
17: aload_2
18: aastore
19: dup
20: iconst_2
21: iload_3
22: invokestatic #598 // Method
scala/runtime/BoxesRunTime.boxToBoolean:(Z)Ljava/lang/Boolean;
25: aastore
26: invokevirtual #602 // Method
org/apache/kyuubi/reflection/DynMethods$UnboundMethod.invoke:(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;
29: checkcast #334 // class
org/apache/arrow/vector/types/pojo/Schema
32: areturn
LineNumberTable:
line 339: 0
line 340: 4
line 341: 13
line 342: 17
line 343: 21
LocalVariableTable:
Start Length Slot Name Signature
0 33 0 this
Lorg/apache/spark/sql/execution/arrow/KyuubiArrowConverters$;
0 33 1 schema Lorg/apache/spark/sql/types/StructType;
0 33 2 timeZone Ljava/lang/String;
0 33 3 errorOnDuplicatedFieldNames Z
MethodParameters:
Name Flags
schema final
timeZone final
errorOnDuplicatedFieldNames final
// the bytecode of the method:
// ```
// private def toArrowSchema(
// schema: StructType,
// timeZone: String,
// errorOnDuplicatedFieldNames: Boolean): ArrowSchema = {
// toArrowSchemaMethod.invoke[ArrowSchema](
// ArrowUtils,
// schema,
// timeZone,
// new java.lang.Boolean(errorOnDuplicatedFieldNames))
// }
// ```
public org.apache.arrow.vector.types.pojo.Schema
org$apache$spark$sql$execution$arrow$KyuubiArrowConverters$$toArrowSchema(org.apache.spark.sql.types.StructType,
java.lang.String, boolean);
descriptor:
(Lorg/apache/spark/sql/types/StructType;Ljava/lang/String;Z)Lorg/apache/arrow/vector/types/pojo/Schema;
flags: ACC_PUBLIC
Code:
stack=8, locals=4, args_size=4
0: aload_0
1: invokespecial #592 // Method
toArrowSchemaMethod:()Lorg/apache/kyuubi/reflection/DynMethods$UnboundMethod;
4: getstatic #166 // Field
org/apache/spark/sql/util/ArrowUtils$.MODULE$:Lorg/apache/spark/sql/util/ArrowUtils$;
7: iconst_3
8: anewarray #4 // class java/lang/Object
11: dup
12: iconst_0
13: aload_1
14: aastore
15: dup
16: iconst_1
17: aload_2
18: aastore
19: dup
20: iconst_2
21: new #577 // class java/lang/Boolean
24: dup
25: iload_3
26: invokespecial #594 // Method
java/lang/Boolean."<init>":(Z)V
29: aastore
30: invokevirtual #598 // Method
org/apache/kyuubi/reflection/DynMethods$UnboundMethod.invoke:(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;
33: checkcast #334 // class
org/apache/arrow/vector/types/pojo/Schema
36: areturn
LineNumberTable:
line 339: 0
line 340: 4
line 341: 13
line 342: 17
line 343: 21
LocalVariableTable:
Start Length Slot Name Signature
0 37 0 this
Lorg/apache/spark/sql/execution/arrow/KyuubiArrowConverters$;
0 37 1 schema Lorg/apache/spark/sql/types/StructType;
0 37 2 timeZone Ljava/lang/String;
0 37 3 errorOnDuplicatedFieldNames Z
MethodParameters:
Name Flags
schema final
timeZone final
errorOnDuplicatedFieldNames final
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
