JingsongLi commented on code in PR #11031:
URL: https://github.com/apache/paimon/pull/11031#discussion_r4561747494
##########
paimon-python/pypaimon/execution/native/read/table_read.py:
##########
@@ -40,6 +40,91 @@
_NATIVE_READ_FILE_SUFFIXES = tuple('.%s' % file_format for file_format in
_NATIVE_READ_FILE_FORMATS)
_NATIVE_BLOB_FILE_SUFFIXES = ('.blob', '.video')
+# Must stay aligned with Rust StringArray downcast and Python
+# ``_KEYTIME_STRING_TYPES``. Kept local so Native does not import the
+# Python execution backend.
+_NATIVE_KEYTIME_STRING_TYPES = frozenset({"CHAR", "VARCHAR", "STRING"})
+_MERGE_MAP_WITH_KEYTIME = "merge_map_with_keytime"
+
+
+def _atomic_type_base_name(field_type):
+ from pypaimon.schema.data_types import AtomicType
+ if not isinstance(field_type, AtomicType):
+ return None
+ raw = field_type.type
+ return raw.split('(', 1)[0].split(' ', 1)[0].upper()
+
+
+def _field_agg_func_name(field_name, primary_keys, options_map,
sequence_fields):
+ # Same precedence as Python ``resolve_agg_func_name`` / Java
+ # ``AggregateMergeFunction.getAggFuncName``, inlined to avoid a
+ # Native → Python execution import.
+ if field_name in sequence_fields:
+ return "last_value"
+ if field_name in primary_keys:
+ return "primary_key"
+ return (
+ options_map.get("fields.{}.aggregate-function".format(field_name))
+ or options_map.get("fields.default-aggregate-function")
+ or "last_non_null_value"
+ )
+
+
+def _native_merge_map_keytime_timestamp_supported(table) -> bool:
+ """Whether Native can run ``merge_map_with_keytime`` for this table.
+
+ Rust resolves the timestamp column at merge time by downcasting to
+ ``StringArray`` and raises ``DataInvalid`` (``requires STRING``,
+ ``source: None``) for any other Arrow type. Only CHAR / VARCHAR /
+ STRING clear that check.
+
+ The failure happens while iterating batches after setup, which
+ bypasses the usual setup-time fallback. Typed timestamps that
+ Python already accepts (TIMESTAMP, DOUBLE, ...) must take the
+ Python aggregator path before Native is opened.
+
+ A non-atomic / unresolvable timestamp type returns ``False`` so
+ Python can raise the precise schema validation error; that is not
+ treated as "unsupported but known".
+ """
+ if not getattr(table, 'is_primary_key_table', False):
+ return True
+ options = table.options
+ if options.merge_engine() != MergeEngine.AGGREGATE:
+ return True
+
+ from pypaimon.schema.data_types import MapType, RowType
+
+ options_map = options.options.to_map()
+ pk_set = set(table.primary_keys)
+ sequence_fields = set(options.sequence_field())
+ for field in table.fields:
+ if _field_agg_func_name(
+ field.name, pk_set, options_map, sequence_fields
+ ) != _MERGE_MAP_WITH_KEYTIME:
+ continue
+ field_type = field.type
+ if (not isinstance(field_type, MapType)
+ or not isinstance(field_type.value, RowType)
+ or not field_type.value.fields):
+ return False
+ row_type = field_type.value
+ ts_field_name = options.field_merge_map_ts_field(field.name)
+ if ts_field_name is None:
+ ts_type = row_type.fields[-1].type
+ else:
+ try:
+ ts_index = row_type.get_field_index(ts_field_name)
+ except ValueError:
+ return False
+ if ts_index < 0:
+ return False
+ ts_type = row_type.fields[ts_index].type
+ # None => type parse / non-atomic failure; not a supported native type.
+ if _atomic_type_base_name(ts_type) not in _NATIVE_KEYTIME_STRING_TYPES:
Review Comment:
[P2] Require matching string comparison before enabling native reads
This gate allows STRING timestamps, but the new Java/Python comparison is
UTF-16 while Rust still compares `&str` in UTF-8 order. With two persisted
versions of the same map key, first at U+E000 and then U+10000, this head's
Python reader keeps the first value and its permitted native reader keeps the
second. Both ASCII controls pass, and native fallback was forbidden, so the
same table's result depends on the execution backend. The current public Rust
`merge_map.rs` still contains this comparison (blob
ff388b217ef86d9420ac35dbda1d6a5088121dba). Fall back for this aggregator until
the native comparator matches, or update/gate native support together with the
semantics change.
##########
paimon-python/pypaimon/execution/python/read/reader/aggregate/aggregators.py:
##########
@@ -1299,8 +1325,19 @@ def _resolve_ts_field_index(self, row_type: RowType,
options: CoreOptions, field
)
)
+ ts_type = row_type.fields[ts_field_index].type
+ ts_base = _atomic_base_name(ts_type)
+ if ts_base not in _KEYTIME_TIMESTAMP_TYPES:
Review Comment:
[P1] Preserve the remaining historically readable timestamp types
The FLOAT/DOUBLE exception does not fully address the earlier
persisted-table compatibility issue. The old Python aggregator also accepted
ordered BINARY/VARBINARY and BOOLEAN fields; this whitelist rejects them before
reading existing files. I independently reproduced all three with filesystem PK
aggregation tables and two committed versions. For BINARY(4)/VARBINARY the
timestamps are ordinary big-endian epoch bytes 9 and 10: the exact merge-base
aggregator reads the newer entry, while this head reopens the identical files
and raises `ValueError`. BOOLEAN false/true has the same regression. Preserve
historical comparison for these types too, or handle a compatibility migration
separately; an extension should not make previously readable tables
inaccessible.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]