Fokko commented on code in PR #4375:
URL: https://github.com/apache/iceberg-python/pull/4375#discussion_r4572510384


##########
pyiceberg/io/pyarrow.py:
##########
@@ -1808,34 +1817,58 @@ def _task_to_record_batches(
             next_index = next_index + len(batch)
             current_index = next_index - len(batch)
             current_batch = batch
+            positions = pa.array(range(current_index, next_index), 
type=pa.int64()) if row_id_index is not None else None
 
             if positional_deletes:
                 # Create the mask of indices that we're interested in
                 indices = _combine_positional_deletes(positional_deletes, 
current_index, current_index + len(batch))
                 current_batch = current_batch.take(indices)
-                if pyarrow_filter is not None:
-                    # Temporary fix until PyArrow 21 is the minimum supported 
version
-                    # (https://github.com/apache/arrow/pull/46057): 
RecordBatch.filter raises
-                    # IndexError on PyArrow <21 when the result is empty; 
Table.filter does not.
-                    table = pa.Table.from_batches([current_batch])
-                    table = table.filter(pyarrow_filter)
-                    if table.num_rows == 0:
-                        current_batch = current_batch.slice(0, 0)
-                    else:
-                        current_batch = table.combine_chunks().to_batches()[0]
+                if positions is not None:
+                    positions = positions.take(indices)
+
+            if filter_after_read and pyarrow_filter is not None:
+                # Temporary fix until PyArrow 21 is the minimum supported 
version
+                # (https://github.com/apache/arrow/pull/46057): 
RecordBatch.filter raises
+                # IndexError on PyArrow <21 when the result is empty; 
Table.filter does not.
+                table = pa.Table.from_batches([current_batch])
+                if positions is not None:
+                    table = table.append_column(_POSITION_COLUMN, positions)
+                table = table.filter(pyarrow_filter)
+                if positions is not None:
+                    positions = table.column(_POSITION_COLUMN).combine_chunks()
+                    table = table.drop_columns([_POSITION_COLUMN])
+                if table.num_rows == 0:
+                    current_batch = current_batch.slice(0, 0)
+                else:
+                    current_batch = table.combine_chunks().to_batches()[0]
 
             # skip empty batches
             if current_batch.num_rows == 0:
                 continue
 
-            yield _to_requested_schema(
+            result_batch = _to_requested_schema(
                 projected_schema,
                 file_project_schema,
                 current_batch,
                 downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
                 projected_missing_fields=projected_missing_fields,
                 allow_timestamp_tz_mismatch=True,
             )
+            if row_id_index is not None and positions is not None:

Review Comment:
   Nit: `positions` is always set when `row_id_index is not None`, so the 
`positions is not None` check is redundant.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to