Github user BryanCutler commented on a diff in the pull request:
https://github.com/apache/spark/pull/21427#discussion_r196242012
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
---
@@ -58,18 +58,18 @@ class ArrowPythonRunner(
new WriterThread(env, worker, inputIterator, partitionIndex, context) {
protected override def writeCommand(dataOut: DataOutputStream): Unit
= {
- PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets)
- if (respectTimeZone) {
- PythonRDD.writeUTF(timeZoneId, dataOut)
- } else {
- dataOut.writeInt(SpecialLengths.NULL)
+ dataOut.writeInt(conf.size)
+ for ((k, v) <- conf) {
+ PythonRDD.writeUTF(k, dataOut)
+ PythonRDD.writeUTF(v, dataOut)
}
+ PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets)
}
protected override def writeIteratorToStream(dataOut:
DataOutputStream): Unit = {
- val arrowSchema = ArrowUtils.toArrowSchema(schema, timeZoneId)
val allocator = ArrowUtils.rootAllocator.newChildAllocator(
s"stdout writer for $pythonExec", 0, Long.MaxValue)
+ val arrowSchema = ArrowUtils.toArrowSchema(schema, timeZoneId)
--- End diff --
change this back, accidental
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
