twuebi commented on code in PR #1250:
URL: https://github.com/apache/iceberg-go/pull/1250#discussion_r3467188213
##########
table/arrow_scanner.go:
##########
@@ -358,38 +355,59 @@ func processPositionalDeletes(ctx context.Context,
deletes set[int64]) recProces
// (GC-friendly) and shared across every per-batch Boolean array; each
// array.NewBoolean / array.NewSlice pair is released after the batch.
//
-// rowCount bounds the mask to the data file's row count. The closure-captured
-// nextIdx tracks absolute position across batches, mirroring
-// processPositionalDeletes.
-func filterByDeletionVector(ctx context.Context, bitmap
*dv.RoaringPositionBitmap, rowCount int64) recProcessFn {
+// rowCount bounds the mask to the data file's row count. cursor maps each
batch
+// row to its original file position: when no row group was pruned the
positions
+// are contiguous and the mask is sliced zero-copy; when pruning skipped groups
+// the emitted rows are non-contiguous, so the keep bits are gathered per row
at
+// the cursor's positions.
+func filterByDeletionVector(ctx context.Context, bitmap
*dv.RoaringPositionBitmap, rowCount int64, cursor *rowPositionCursor)
recProcessFn {
nextIdx := int64(0)
keepBits := bitmap.KeepMaskBytes(rowCount)
buf := memory.NewBufferBytes(keepBits)
+ mem := compute.GetAllocator(ctx)
return func(r arrow.RecordBatch) (arrow.RecordBatch, error) {
defer r.Release()
- currentIdx := nextIdx
- nextIdx += r.NumRows()
+ nrows := r.NumRows()
+
+ if !cursor.src.pruned() {
+ currentIdx := nextIdx
+ nextIdx += nrows
+
+ // Wrap (and slice) the shared keep-mask buffer for
this batch.
+ // array.NewSlice on a Boolean array tracks the
bit-level offset,
+ // so we don't need byte-aligned slicing — currentIdx
can land
+ // anywhere within a byte.
+ full := array.NewBoolean(int(rowCount), buf, nil, 0)
+ defer full.Release()
+ sliced := array.NewSlice(full, currentIdx,
nextIdx).(*array.Boolean)
+ defer sliced.Release()
+
+ return compute.FilterRecordBatch(ctx, r, sliced,
compute.DefaultFilterOptions())
+ }
- // Wrap (and slice) the shared keep-mask buffer for this batch.
- // array.NewSlice on a Boolean array tracks the bit-level
offset,
- // so we don't need byte-aligned slicing — currentIdx can land
- // anywhere within a byte.
- full := array.NewBoolean(int(rowCount), buf, nil, 0)
- defer full.Release()
- sliced := array.NewSlice(full, currentIdx,
nextIdx).(*array.Boolean)
- defer sliced.Release()
+ bldr := array.NewBooleanBuilder(mem)
+ defer bldr.Release()
+ bldr.Reserve(int(nrows))
+ for range nrows {
+ pos := cursor.next()
+ // Test keep bit at absolute position pos: byte pos/8,
bit pos%8,
+ // LSB-first (the layout the fast path's
array.NewBoolean reads).
+ bldr.Append(keepBits[pos>>3]&(1<<(uint(pos)&7)) != 0)
Review Comment:
added the guard, thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
