(datafusion-comet) branch main updated: Move checking parquet types above fetching the batch from native code. (#1809)

[parthc]

This is an automated email from the ASF dual-hosted git repository.

parthc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new d59883374 Move checking parquet types above fetching the batch from 
native code. (#1809)
d59883374 is described below

commit d5988337437ebbf6bb6a4c0bc5f64d35926e65d2
Author: Matt Butrovich <mbutrov...@users.noreply.github.com>
AuthorDate: Wed May 28 20:32:06 2025 -0400

    Move checking parquet types above fetching the batch from native code. 
(#1809)
---
 .../main/java/org/apache/comet/parquet/NativeBatchReader.java | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git 
a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java 
b/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
index 7a6a1d714..51ba97279 100644
--- a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
@@ -321,8 +321,6 @@ public class NativeBatchReader extends RecordReader<Void, 
ColumnarBatch> impleme
       }
       long[] starts = new long[blocks.size()];
       long[] lengths = new long[blocks.size()];
-      starts = new long[blocks.size()];
-      lengths = new long[blocks.size()];
       int blockIndex = 0;
       for (BlockMetaData block : blocks) {
         long blockStart = block.getStartingPos();
@@ -613,7 +611,10 @@ public class NativeBatchReader extends RecordReader<Void, 
ColumnarBatch> impleme
 
   @SuppressWarnings("deprecation")
   private int loadNextBatch() throws Throwable {
-    long startNs = System.nanoTime();
+
+    for (ParquetColumn childColumn : 
JavaConverters.seqAsJavaList(parquetColumn.children())) {
+      checkParquetType(childColumn);
+    }
 
     int batchSize = Native.readNextRecordBatch(this.handle);
     if (batchSize == 0) {
@@ -622,10 +623,6 @@ public class NativeBatchReader extends RecordReader<Void, 
ColumnarBatch> impleme
     if (importer != null) importer.close();
     importer = new CometSchemaImporter(ALLOCATOR);
 
-    for (ParquetColumn childColumn : 
JavaConverters.seqAsJavaList(parquetColumn.children())) {
-      checkParquetType(childColumn);
-    }
-
     List<Type> fields = requestedSchema.getFields();
     for (int i = 0; i < fields.size(); i++) {
       if (!missingColumns[i]) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org