zhztheplayer commented on code in PR #6646:
URL: https://github.com/apache/incubator-gluten/pull/6646#discussion_r1698026881
##########
cpp/core/jni/JniWrapper.cc:
##########
@@ -1093,22 +1093,16 @@ JNIEXPORT void JNICALL
Java_org_apache_gluten_datasource_DatasourceJniWrapper_cl
JNI_METHOD_END()
}
-JNIEXPORT void JNICALL
Java_org_apache_gluten_datasource_DatasourceJniWrapper_write( // NOLINT
+JNIEXPORT void JNICALL
Java_org_apache_gluten_datasource_DatasourceJniWrapper_writeBatch( // NOLINT
JNIEnv* env,
jobject wrapper,
jlong dsHandle,
- jobject jIter) {
+ jlong batchHandler) {
Review Comment:
nit: `batchHandle` rather than `batchHandler`
##########
backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala:
##########
@@ -74,29 +72,26 @@ trait VeloxFormatWriterInjects extends
GlutenFormatWriterInjectsBase {
cSchema.close()
}
- // FIXME: remove this once we support push-based write.
- val queueSize =
context.getConfiguration.getInt(GlutenConfig.VELOX_WRITER_QUEUE_SIZE.key, 64)
-
- val writeQueue =
- new VeloxWriteQueue(
- TaskResources.getLocalTaskContext(),
- dsHandle,
- arrowSchema,
- allocator,
- datasourceJniWrapper,
- filePath,
- queueSize)
-
new OutputWriter {
override def write(row: InternalRow): Unit = {
val batch = row.asInstanceOf[FakeRow].batch
Preconditions.checkState(ColumnarBatches.isLightBatch(batch))
ColumnarBatches.retain(batch)
- writeQueue.enqueue(batch)
+ val batchHandler = {
+ if (batch.numCols == 0) {
+ // the operation will find a zero column batch from a task-local
pool
+
ColumnarBatchJniWrapper.create(runtime).getForEmptySchema(batch.numRows)
+ } else {
+ val offloaded =
+
ColumnarBatches.ensureOffloaded(ArrowBufferAllocators.contextInstance, batch)
+ ColumnarBatches.getNativeHandle(offloaded)
+ }
+ }
+ datasourceJniWrapper.writeBatch(dsHandle, batchHandler)
Review Comment:
ditto
##########
gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java:
##########
@@ -53,7 +52,7 @@ public long nativeInitDatasource(String filePath, long
cSchema, Map<String, Stri
public native void close(long dsHandle);
- public native void write(long dsHandle, ColumnarBatchInIterator iterator);
+ public native void writeBatch(long dsHandle, long batchHandler);
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
