JingsongLi commented on code in PR #9486:
URL: https://github.com/apache/paimon/pull/9486#discussion_r3892207929
##########
paimon-python/pypaimon/read/datasource/torch_dataset.py:
##########
@@ -152,10 +257,97 @@ def __getitem__(self, index: int):
Returns:
Dictionary containing the row data
"""
- if not self._data:
+ if len(self) == 0:
return None
+ if isinstance(index, slice):
+ return self.__getitems__(range(*index.indices(len(self))))
+ return self.__getitems__([index])[0]
+
+ def __getitems__(self, indices) -> List[dict]:
+ normalized = [self._normalize_index(index) for index in indices]
+ if not normalized:
+ return []
+ if self._row_ids is None:
+ return self._data.take(pa.array(
+ normalized, type=pa.int64())).to_pylist()
+
+ if isinstance(self._row_ids, _RowIdRangeIndex):
+ row_ids = self._row_ids.take(normalized)
+ else:
+ row_ids = self._row_ids.take(pa.array(
+ normalized, type=pa.int64())).to_pylist()
+ ranges = Range.sort_and_merge_overlap(
+ [Range(row_id, row_id) for row_id in set(row_ids)], True)
+ splits = self._select_splits(ranges)
+
+ output_has_row_id = any(
+ field.name == SpecialFields.ROW_ID.name
+ for field in self.table_read.read_type
+ )
+ read_type = list(self.table_read.read_type)
+ if not output_has_row_id:
+ read_type.append(SpecialFields.ROW_ID)
+ batch_read = TableRead(
Review Comment:
[P2] Keep the routing row ID outside authorization masking
`batch_read` reuses the original `QueryAuthSplit`s after appending `_ROW_ID`
to the projection. If the authorization result contains a masking rule for
`_ROW_ID` (even when `_ROW_ID` was not user-projected; `AuthMaskingReader`
intentionally ignores absent targets), the internal ID is now masked before the
rows are indexed below. The previous eager map-style read succeeds because
`_ROW_ID` stays absent, but this lazy path loses the routing keys and raises
`Paimon rows disappeared while reading TorchDataset`. I reproduced this on a
row-tracking/data-evolution table with a `NULL` mask for `_ROW_ID`. Please
obtain the routing ID before masking, or otherwise prevent masking from
applying to this internal column.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
