Stephen0421 commented on code in PR #9148:
URL: https://github.com/apache/paimon/pull/9148#discussion_r3773752586
##########
paimon-python/pypaimon/read/split_read.py:
##########
@@ -1003,6 +1047,23 @@ def _build_merge_function(self):
value_field_names=[f.name for f in self.value_fields],
)
+ def _create_blob_view_prescan_reader(self, field_names: set):
+ value_fields = self.read_fields[-self.value_arity:]
+ prescan_fields = [f for f in value_fields if f.name in field_names]
+ if not prescan_fields:
+ return EmptyFileRecordReader()
Review Comment:
Fixed. An empty view projection now returns `EmptyRecordBatchReader`. A
non-empty merge prescan wraps the row reader with `RecordReaderToBatchAdapter`
so `BlobInlineConvertReader` can call `read_arrow_batch()`. Added projection
tests, including convert + empty prescan reading the main batch.
##########
paimon-python/pypaimon/table/row/offset_row.py:
##########
@@ -55,12 +62,57 @@ def get_field(self, pos: int):
raise IndexError(f"Position {pos} is out of bounds for row arity
{self.arity}")
return self.row_tuple[self.offset + pos]
- def get_blob(self, pos: int):
+ @staticmethod
+ def _normalize_blob_bytes(value):
+ if value is None:
+ return None
+ if hasattr(value, 'as_py'):
+ value = value.as_py()
+ if isinstance(value, str):
+ value = value.encode('utf-8')
+ if isinstance(value, bytearray):
+ value = bytes(value)
+ return value
+
+ def _resolve_blob_view_struct(self, view_struct):
from pypaimon.table.row.blob import Blob
+ if self._blob_view_lookup is not None:
+ if self._blob_view_lookup.resolve_to_null(view_struct):
+ return None
+ descriptor = self._blob_view_lookup.resolve_descriptor(view_struct)
+ uri_reader = self._blob_view_lookup.resolve_uri_reader(view_struct)
+ return Blob.from_descriptor(uri_reader, descriptor)
+ return Blob.from_view(view_struct)
+
+ def _blob_from_descriptor_field_bytes(self, raw: bytes):
+ from pypaimon.table.row.blob import Blob, BlobDescriptor
+
+ if BlobDescriptor.is_blob_descriptor(raw):
+ return Blob.from_descriptor_bytes(raw, self._file_io)
+ if BlobDescriptor.parse_if_serialized(raw) is not None:
+ return Blob.from_descriptor_bytes(raw, self._file_io)
+ try:
+ # Accept v1/v2 descriptors with trailing padding (Java
deserialize).
+ return Blob.from_descriptor_bytes(raw, self._file_io)
+ except ValueError:
Review Comment:
Agreed. Descriptor-indexed values now use `from_descriptor_bytes` strictly;
truncated or corrupt bytes raise `ValueError` instead of falling back to
`BlobData`. Materialized payloads are represented by clearing
`descriptor_field_indices` in `BlobInlineConvertReader` when
`blob-as-descriptor=false`. Added a truncated-v1 rejection test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
