PyRSA commented on code in PR #11031:
URL: https://github.com/apache/paimon/pull/11031#discussion_r4638726014
##########
paimon-python/pypaimon/execution/native/read/table_read.py:
##########
@@ -40,6 +40,75 @@
_NATIVE_READ_FILE_SUFFIXES = tuple('.%s' % file_format for file_format in
_NATIVE_READ_FILE_FORMATS)
_NATIVE_BLOB_FILE_SUFFIXES = ('.blob', '.video')
+# merge_map_with_keytime is unsafe on Native until Rust matches Java
+# UTF-16 string order (e.g. U+E000 vs U+10000):
+# * AGGREGATE: fall back to the Python aggregator before Native opens.
+# * PARTIAL_UPDATE (incl. sequence-group field aggregators): Python
+# cannot run this config either, so raise before Native opens rather
+# than returning a wrong UTF-8 winner.
+_MERGE_MAP_WITH_KEYTIME = "merge_map_with_keytime"
+
+
+def _field_agg_func_name(field_name, primary_keys, options_map,
sequence_fields):
+ # Same precedence as Python ``resolve_agg_func_name`` / Java
+ # ``AggregateMergeFunction.getAggFuncName``, inlined to avoid a
+ # Native → Python execution import.
+ if field_name in sequence_fields:
+ return "last_value"
+ if field_name in primary_keys:
+ return "primary_key"
+ return (
+ options_map.get("fields.{}.aggregate-function".format(field_name))
+ or options_map.get("fields.default-aggregate-function")
+ or "last_non_null_value"
+ )
+
+
+def _table_uses_merge_map_keytime(table) -> bool:
+ """True when a field aggregator actually resolves to keytime.
+
+ Covers Aggregate and Partial Update (including sequence-group tables
+ that set ``fields.<f>.aggregate-function=merge_map_with_keytime`` on
+ a group member). Detection follows field-aggregator precedence; it
+ does not treat ``sequence-group`` alone as keytime.
+ """
+ if not getattr(table, 'is_primary_key_table', False):
+ return False
+ options = table.options
+ if options.merge_engine() not in (
+ MergeEngine.AGGREGATE, MergeEngine.PARTIAL_UPDATE):
+ return False
+
+ options_map = options.options.to_map()
+ pk_set = set(table.primary_keys)
+ sequence_fields = set(options.sequence_field())
+ for field in table.fields:
+ if _field_agg_func_name(
+ field.name, pk_set, options_map, sequence_fields
+ ) == _MERGE_MAP_WITH_KEYTIME:
+ return True
+ return False
+
+
+def _native_merge_map_keytime_timestamp_supported(table) -> bool:
+ """Whether Native can run ``merge_map_with_keytime`` for this table.
+
+ Always ``False`` when any field uses ``merge_map_with_keytime`` on a
+ PK aggregation or partial-update table. Non-PK / other engines /
+ other aggregators are unchanged and still clear this gate.
+ """
+ return not _table_uses_merge_map_keytime(table)
Review Comment:
Fixed by narrowing the Native read gate to effective STRING keytime fields
that are actually accessed by the current read. Keytime fields omitted from
both the projection and predicate no longer disable otherwise supported Native
operations.
Added a persisted-table regression with a BIGINT-keytime MAP and VARIANT
payload. The test verifies that `variant_get(payload, '$.score', 'float')`
stays on the Native read path and returns the expected latest score without
falling back to the Python reader.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]