HeartSaVioR commented on code in PR #37864:
URL: https://github.com/apache/spark/pull/37864#discussion_r970207233
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowInput.scala:
##########
@@ -26,21 +26,30 @@ import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.api.python.{BasePythonRunner, PythonRDD}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.arrow.ArrowWriter
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.ArrowUtils
import org.apache.spark.util.Utils
/**
* A trait that can be mixed-in with [[BasePythonRunner]]. It implements the
logic from
- * JVM (an iterator of internal rows) to Python (Arrow).
+ * JVM (an iterator of internal rows + additional data if required) to Python
(Arrow).
*/
-private[python] trait PythonArrowInput { self:
BasePythonRunner[Iterator[InternalRow], _] =>
+private[python] trait PythonArrowInput[IN] { self: BasePythonRunner[IN, _] =>
+ protected val sqlConf = SQLConf.get
Review Comment:
Ah OK didn't notice this. Will remove this. Thanks for the pointer.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowInput.scala:
##########
@@ -107,3 +107,27 @@ private[python] trait PythonArrowInput { self:
BasePythonRunner[Iterator[Interna
}
}
}
+
+private[python] trait BasicPythonArrowInput extends
PythonArrowInput[Iterator[InternalRow]] {
+ self: BasePythonRunner[Iterator[InternalRow], _] =>
+
+ protected def writeIteratorToArrowStream(
+ root: VectorSchemaRoot,
+ writer: ArrowStreamWriter,
+ dataOut: DataOutputStream,
+ inputIterator: Iterator[Iterator[InternalRow]]): Unit = {
Review Comment:
Nice finding!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
