JingsongLi commented on code in PR #10723:
URL: https://github.com/apache/paimon/pull/10723#discussion_r4494168913


##########
paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexRecordsReader.java:
##########
@@ -21,195 +21,455 @@
 import org.apache.paimon.annotation.VisibleForTesting;
 import org.apache.paimon.arrow.reader.ArrowBatchReader;
 import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.columnar.AllNullColumnVector;
+import org.apache.paimon.data.columnar.ColumnVector;
+import org.apache.paimon.data.columnar.ColumnarRow;
+import org.apache.paimon.data.columnar.VectorizedColumnBatch;
+import org.apache.paimon.data.columnar.VectorizedRowIterator;
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.reader.FileRecordIterator;
 import org.apache.paimon.reader.FileRecordReader;
 import org.apache.paimon.table.SpecialFields;
 import org.apache.paimon.types.DataField;
 import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.ExecutorThreadFactory;
 import org.apache.paimon.utils.LongIterator;
-import org.apache.paimon.utils.ProjectedRow;
+import org.apache.paimon.utils.RoaringBitmap32;
 
 import dev.vortex.api.DataSource;
 import dev.vortex.api.Expression;
 import dev.vortex.api.ImmutableScanOptions;
 import dev.vortex.api.Partition;
 import dev.vortex.api.Scan;
 import dev.vortex.api.ScanOptions;
-import dev.vortex.api.Session;
 import dev.vortex.arrow.ArrowAllocation;
+import dev.vortex.arrow.ArrowBatch;
+import dev.vortex.arrow.ArrowBatchStream;
+import dev.vortex.arrow.ViewColumn;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.ipc.ArrowReader;
 
 import javax.annotation.Nullable;
 
+import java.io.Closeable;
 import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
-/** File reader for Vortex format. */
+/**
+ * File reader for Vortex format.
+ *
+ * <p>Each Arrow batch from the native scan is a new root that this reader 
owns until the consumer
+ * releases the batch, so a later batch never overwrites one that the consumer 
still holds. Batches
+ * are returned as {@link VectorizedRowIterator}s, so engines can consume them 
column by column.
+ *
+ * <p>A pushed-down filter or a row selection drops rows in native code. The 
file position of each
+ * returned row then comes from the Vortex row index, because deletion vectors 
and row tracking
+ * depend on it.
+ */
 public class VortexRecordsReader implements FileRecordReader<InternalRow> {
 
-    private final ArrowBatchReader arrowBatchReader;
+    private static final String ROW_INDEX_FIELD = "_PAIMON_VORTEX_ROW_INDEX";
+
     private final Path filePath;
     private final BufferAllocator allocator;
-    private final Session session;
+    @Nullable private final Closeable readable;
     private final DataSource dataSource;
     private final Scan scan;
-    private final LongIterator positionIterator;
+    private final ArrowBatchReader arrowBatchReader;
     @Nullable private final int[] physicalFieldMapping;
-    private ArrowReader currentArrowReader;
-    private Partition currentPartition;
-    private long returnedPosition = -1;
-
-    public VortexRecordsReader(
-            Path path,
+    private final boolean readsRowIndex;
+    private final Set<ArrowBatch> activeBatches = new HashSet<>();
+
+    /** Fetches the next batch while the consumer processes the current one. */
+    private static final ExecutorService PREFETCH_POOL =
+            Executors.newCachedThreadPool(new 
ExecutorThreadFactory("vortex-batch-prefetch"));
+
+    // Only one thread at a time calls fetchNext: the caller, or the prefetch 
it waits for.
+    @Nullable private ArrowBatchStream currentStream;
+    @Nullable private Future<ArrowBatch> prefetched;
+    private long nextPosition;
+    private boolean closed;
+
+    VortexRecordsReader(
+            VortexInput input,
+            Path filePath,
             RowType dataSchemaRowType,
             RowType projectedRowType,
-            @Nullable long[] rowIndices,
-            @Nullable Expression predicate,
-            Map<String, String> storageOptions) {
-        this.filePath = path;
+            @Nullable RoaringBitmap32 selection,
+            @Nullable Expression filter) {
+        this.filePath = filePath;
+
         RowType physicalReadRowType = physicalReadRowType(dataSchemaRowType, 
projectedRowType);
         this.physicalFieldMapping =
                 physicalFieldMapping(dataSchemaRowType, physicalReadRowType, 
projectedRowType);
+        this.arrowBatchReader =
+                new ArrowBatchReader(physicalReadRowType, true, 
VortexArrowConverters.INSTANCE);
+
+        List<String> columns = physicalReadRowType.getFieldNames();
+        this.readsRowIndex = filter != null || selection != null || 
columns.isEmpty();
+
         this.allocator =
                 ArrowAllocation.rootAllocator()
                         .newChildAllocator("vortex-reader", 0, Long.MAX_VALUE);
 
+        Closeable openedReadable = null;
+        DataSource openedDataSource = null;
         try {
-            this.session = Session.create();
-            try {
-                this.dataSource = DataSource.open(session, 
path.toUri().toString(), storageOptions);
-                try {
-                    ImmutableScanOptions.Builder scanBuilder =
-                            ImmutableScanOptions.builder().ordered(true);
-
-                    java.util.List<String> columns = 
physicalReadRowType.getFieldNames();
-                    // The scan clones the projection it is given, so both the 
root and the select
-                    // built here are ours to release once dataSource.scan has 
returned.
-                    try (Expression root = Expression.root();
-                            Expression projection =
-                                    Expression.select(columns.toArray(new 
String[0]), root)) {
-                        scanBuilder.projection(projection);
-
-                        if (rowIndices != null) {
-                            scanBuilder.selectionIndices(rowIndices);
-                            
scanBuilder.selectionMode(ScanOptions.SelectionMode.INCLUDE);
-                        }
-                        if (predicate != null) {
-                            scanBuilder.filter(predicate);
-                        }
-                        this.scan = dataSource.scan(scanBuilder.build());
-                    }
-                } catch (Exception e) {
-                    dataSource.close();
-                    throw e;
-                }
-            } catch (Exception e) {
-                session.close();
-                throw e;
+            openedReadable = input.readable();
+            openedDataSource = input.open();
+            // The scan clones the expressions it is given, so the projection 
is ours to release
+            // once the scan exists.
+            try (Expression projection = projection(columns)) {
+                this.scan = openedDataSource.scan(scanOptions(projection, 
selection, filter));
             }
-        } catch (Exception e) {
-            allocator.close();
-            throw new RuntimeException(e);
+        } catch (Throwable t) {
+            closeQuietly(openedDataSource, t);
+            closeQuietly(openedReadable, t);
+            closeQuietly(allocator, t);
+            throw t;
         }
 
-        this.arrowBatchReader = new ArrowBatchReader(physicalReadRowType, 
true);
-        long totalRowCount = dataSource.rowCount();
-        this.positionIterator =
-                rowIndices != null
-                        ? LongIterator.fromArray(rowIndices)
-                        : LongIterator.fromRange(0, totalRowCount > 0 ? 
totalRowCount : 0);
+        this.readable = openedReadable;
+        this.dataSource = openedDataSource;
+
+        // Start decoding the first batch now. A merge opens all files of a 
split before reading
+        // any of them, so their first batches then decode in parallel.
+        this.prefetched = PREFETCH_POOL.submit(this::fetchNext);
     }
 
     @Nullable
     @Override
     public FileRecordIterator<InternalRow> readBatch() throws IOException {
-        // Try to load the next batch from the current ArrowReader
-        if (currentArrowReader != null && currentArrowReader.loadNextBatch()) {
-            return toBatchIterator(currentArrowReader.getVectorSchemaRoot());
+        while (true) {
+            ArrowBatch batch = takeNext();
+            if (batch == null) {
+                return null;
+            }
+
+            track(batch);
+            if (batch.rowCount() > 0) {
+                prefetched = PREFETCH_POOL.submit(this::fetchNext);
+                return toBatchIterator(batch);
+            }
+            releaseBatch(batch);
         }
+    }
 
-        // Close current reader and move to the next partition
-        closeCurrentArrowReader();
+    @Nullable
+    private ArrowBatch takeNext() throws IOException {
+        if (prefetched == null) {
+            return fetchNext();
+        }
 
-        while (scan.hasNext()) {
-            closeCurrentPartition();
-            currentPartition = scan.next();
-            currentArrowReader = currentPartition.scanArrow(allocator);
-            if (currentArrowReader.loadNextBatch()) {
-                return 
toBatchIterator(currentArrowReader.getVectorSchemaRoot());
-            }
-            closeCurrentArrowReader();
+        // Forget the prefetch only after its result or its failure is taken. 
An interrupted wait
+        // keeps it, also when it has just completed: it can still run, or 
hold a batch that only
+        // close can then release.
+        ArrowBatch batch;
+        try {
+            batch = prefetched.get();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            InterruptedIOException interrupted =
+                    new InterruptedIOException("Interrupted while reading a 
Vortex batch");
+            interrupted.initCause(e);
+            throw interrupted;
+        } catch (ExecutionException e) {
+            prefetched = null;
+            throw unwrap(e);
         }
 
-        return null;
+        prefetched = null;
+        return batch;
     }
 
-    private FileRecordIterator<InternalRow> toBatchIterator(VectorSchemaRoot 
vsr) {
-        Iterator<InternalRow> rows = 
arrowBatchReader.readBatch(vsr).iterator();
-        ProjectedRow projectedRow =
-                physicalFieldMapping == null ? null : 
ProjectedRow.from(physicalFieldMapping);
+    /** Reads the next batch, moving on to the next partition when one is 
exhausted. */
+    @Nullable
+    private ArrowBatch fetchNext() throws IOException {
+        while (true) {
+            if (currentStream != null) {
+                ArrowBatch batch = currentStream.next();
+                if (batch != null) {
+                    return batch;
+                }
+                closeCurrentStream();
+            }
 
-        return new FileRecordIterator<InternalRow>() {
-            @Override
-            public long returnedPosition() {
-                return returnedPosition;
+            if (!scan.hasNext()) {
+                return null;
             }
 
-            @Override
-            public Path filePath() {
-                return filePath;
+            try (Partition partition = scan.next()) {
+                currentStream = partition.scanArrow(allocator);
             }
+        }
+    }
 
-            @Nullable
-            @Override
-            public InternalRow next() {
-                if (!rows.hasNext()) {
-                    return null;
+    /**
+     * Waits for the prefetch to complete, also when the thread is 
interrupted, because native IO in
+     * the prefetch can still use the stream and the allocator. The interrupt 
flag is restored after
+     * the wait.
+     */
+    @Nullable
+    private static ArrowBatch awaitUninterruptibly(Future<ArrowBatch> future) 
throws IOException {
+        boolean interrupted = false;
+        try {
+            while (true) {
+                try {
+                    return future.get();
+                } catch (InterruptedException e) {
+                    interrupted = true;
+                } catch (ExecutionException e) {
+                    throw unwrap(e);
                 }
-                returnedPosition = positionIterator.next();
-                InternalRow row = rows.next();
-                return projectedRow == null ? row : 
projectedRow.replaceRow(row);
             }
+        } finally {
+            if (interrupted) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    }
 
-            @Override
-            public void releaseBatch() {}
-        };
+    private static IOException unwrap(ExecutionException e) {
+        Throwable cause = e.getCause();
+        if (cause instanceof IOException) {
+            return (IOException) cause;
+        }
+        if (cause instanceof RuntimeException) {
+            throw (RuntimeException) cause;
+        }
+        if (cause instanceof Error) {
+            throw (Error) cause;
+        }
+        return new IOException(cause);
     }
 
-    private void closeCurrentArrowReader() {
-        if (currentArrowReader != null) {
+    @Override
+    public void close() throws IOException {
+        if (closed) {
+            return;
+        }
+        closed = true;
+
+        Throwable failure = null;
+        if (prefetched != null) {
+            // Wait for the prefetch, which may still use the stream, before 
closing anything.
+            Future<ArrowBatch> future = prefetched;
+            prefetched = null;
             try {
-                currentArrowReader.close();
-            } catch (IOException e) {
-                // ignore
+                ArrowBatch batch = awaitUninterruptibly(future);
+                if (batch != null) {
+                    batch.close();
+                }
+            } catch (Throwable t) {
+                failure = t;
+            }
+        }
+        try {
+            closeCurrentStream();
+        } catch (Throwable t) {
+            if (failure == null) {
+                failure = t;
+            } else {
+                failure.addSuppressed(t);
+            }
+        }
+
+        List<ArrowBatch> unreleased;
+        synchronized (activeBatches) {
+            unreleased = new ArrayList<>(activeBatches);
+            activeBatches.clear();
+        }
+        for (ArrowBatch batch : unreleased) {
+            failure = closeQuietly(batch, failure);
+        }
+
+        failure = closeQuietly(scan, failure);
+        failure = closeQuietly(dataSource, failure);
+        failure = closeQuietly(readable, failure);
+        failure = closeQuietly(allocator, failure);
+
+        if (failure instanceof IOException) {
+            throw (IOException) failure;
+        } else if (failure instanceof RuntimeException) {
+            throw (RuntimeException) failure;
+        } else if (failure instanceof Error) {
+            throw (Error) failure;
+        } else if (failure != null) {
+            throw new IOException(failure);
+        }
+    }
+
+    private static ScanOptions scanOptions(
+            Expression projection,
+            @Nullable RoaringBitmap32 selection,
+            @Nullable Expression filter) {
+        // Paimon merges sorted files and maps deletion vectors by position, 
so rows must keep
+        // their file order.
+        ImmutableScanOptions.Builder builder = 
ImmutableScanOptions.builder().ordered(true);
+
+        if (selection != null) {
+            builder.selectionRoaringBitmap(serializeSelection(selection))
+                    .selectionMode(ScanOptions.SelectionMode.INCLUDE_ROARING);
+        }
+
+        if (filter != null) {
+            builder.filter(filter);
+        }
+
+        return builder.projection(projection).build();
+    }
+
+    private Expression projection(List<String> columns) {
+        if (!readsRowIndex) {
+            try (Expression root = Expression.root()) {
+                return Expression.select(columns.toArray(new String[0]), root);
+            }
+        }
+
+        String[] names = new String[columns.size() + 1];
+        Expression[] values = new Expression[columns.size() + 1];
+        try {
+            for (int i = 0; i < columns.size(); i++) {
+                names[i] = columns.get(i);
+                values[i] = Expression.column(columns.get(i));
+            }
+            names[columns.size()] = ROW_INDEX_FIELD;

Review Comment:
   [P2] Choose a row-index alias that cannot collide with projected user columns
   
   `_PAIMON_VORTEX_ROW_INDEX` is a legal user field: schema validation only 
reserves the existing SpecialFields names and key prefix. With a committed 
Vortex append table `{id INT, _PAIMON_VORTEX_ROW_INDEX BIGINT}`, the unfiltered 
read returns all ten correct rows, but `id >= 3` with `executeFilter()` throws 
`ClassCastException: BigIntVector cannot be cast to UInt8Vector` at line 390. 
The packed projection contains both the user field and the synthetic row 
position under this name, and name-based lookup retrieves the user vector. The 
identical public-table case passes on the baseline. Please choose a 
collision-free alias and retain its identity through projection/position 
lookup. This code is unchanged in the 0.93 bump; the executable reproduction 
used 0.92 because the 0.93 artifacts are not yet resolvable.



##########
paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexPredicateConverter.java:
##########
@@ -143,10 +157,181 @@ public Expression visit(CompoundPredicate predicate) {
         }
     }
 
+    @Nullable
+    private static Expression.BinaryOp binaryOp(LeafFunction function) {
+        if (function instanceof Equal) {
+            return Expression.BinaryOp.EQ;
+        } else if (function instanceof NotEqual) {
+            return Expression.BinaryOp.NOT_EQ;
+        } else if (function instanceof GreaterThan) {
+            return Expression.BinaryOp.GT;
+        } else if (function instanceof GreaterOrEqual) {
+            return Expression.BinaryOp.GTE;
+        } else if (function instanceof LessThan) {
+            return Expression.BinaryOp.LT;
+        } else if (function instanceof LessOrEqual) {
+            return Expression.BinaryOp.LTE;
+        }
+        return null;
+    }
+
+    /**
+     * {@code IN} and {@code NOT IN} with SQL null rules, which Paimon follows 
too: a null field
+     * never matches, and a null in the list makes {@code NOT IN} false for 
every row.
+     */
+    @Nullable
+    private static Expression in(
+            Expression field, DataType type, List<Object> literals, boolean 
negated) {
+        List<Expression> elements = new ArrayList<>();
+        Expression elementType = null;
+        Expression list = null;
+        try {
+            elementType = nullLiteral(type);
+            if (elementType == null) {
+                return null;
+            }
+
+            for (Object literal : literals) {
+                Expression element = literal == null ? nullLiteral(type) : 
toLiteral(type, literal);
+                if (element == null) {
+                    return null;
+                }
+                elements.add(element);
+            }
+
+            list = Expression.literalList(elementType, true, 
elements.toArray(new Expression[0]));
+            return negated ? Expression.notIn(field, list) : 
Expression.in(field, list);

Review Comment:
   [P2] Preserve Paimon NaN equality when pushing floating IN lists
   
   A large DOUBLE IN list containing `Double.NaN` now drops matching NaNs with 
other signs/payloads. Paimon In.test uses Double.compare (all NaNs compare 
equal), whereas Vortex membership compares their raw bits. In a real committed 
append table with canonical, payload and negative NaNs, a 26-literal IN query 
followed by executeFilter() returns only the canonical row instead of all 
three; unfiltered reads retain all bits, and the same case returns all three on 
the baseline where this leaf is not pushed. Spark forwards large IN lists to 
this path too. The [0.93 native 
implementation](https://github.com/vortex-data/vortex/blob/0.93.0/vortex-array/src/array/haystack/mod.rs#L164-L180)
 still explicitly uses bit-pattern membership, and this converter is unchanged 
by the dependency bump. Please decline floating IN pushdown when NaN is 
present, or normalize native membership to Paimon equality, and cover 
noncanonical/negative NaNs.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to