JingsongLi commented on code in PR #11031:
URL: https://github.com/apache/paimon/pull/11031#discussion_r4588751499
##########
paimon-python/pypaimon/execution/native/read/table_read.py:
##########
@@ -114,6 +183,15 @@ def read_batches(
if not self._blob_view_supported():
return self._fallback(
"the table uses an unsupported BLOB view")
+ # Keytime: Aggregate falls back to Python; Partial Update has no
+ # correct Python path either, so fail before Native opens.
+ if not _native_merge_map_keytime_timestamp_supported(read.table):
Review Comment:
[P2] Apply the keytime capability check to Native MERGE table sources
This only protects NativeTableRead. create_native_merge_into still wraps an
aggregation/partial-update PK source with create_native_write_table and lets
Rust read it internally, bypassing this gate. With real Rust 0.5.0, I persisted
two source versions for the same PK/map key: U+E000/first, then U+10000/second.
The public source read now returns first, but Native MERGE INTO a DE target
using that table as source writes second. The identical Native MERGE using the
public read's materialized Arrow source writes first; Python MERGE matching was
forbidden in both probes. On the exact merge-base both source forms write
second, confirming the new semantic disagreement. Please qualify table sources
with the same capability contract before Native staging and use a correct
logical source route or reject unsupported partial-update sources explicitly.
Cover the committed target result, since the ordinary reader fallback cannot
repair this write.
##########
paimon-python/pypaimon/tests/native_merge_map_keytime_fallback_test.py:
##########
@@ -0,0 +1,555 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Native gates for merge_map_with_keytime (UTF-8 vs UTF-16).
+
+* Aggregate Reader: fall back to Python before Native opens.
+* Partial Update Reader: raise before Native opens (Python unsupported).
+* Writer: refuse Native so backend_dispatch selects Python Writer.
+Non-keytime aggregators keep Native.
+"""
+
+from unittest.mock import Mock, patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.common.options.core_options import CoreOptions
+from pypaimon.common.options.options import Options
+from pypaimon.execution.native.read.table_read import (
+ NativeTableRead,
+ _native_merge_map_keytime_timestamp_supported,
+ _table_uses_merge_map_keytime,
+)
+from pypaimon.execution.native.write.native_write import (
+ _native_write_refuses_merge_map_keytime,
+ create_native_write,
+)
+from pypaimon.execution.python.write.table_write import PythonTableWrite
+from pypaimon.schema.data_types import AtomicType, DataField, MapType, RowType
+from pypaimon.write.table_write import BatchTableWrite
+from pypaimon.write.write_dispatch import NativeTableWrite
+
+
+def _options(**extra):
+ data = {
+ 'bucket': '1',
+ 'merge-engine': 'aggregation',
+ 'file.format': 'parquet',
+ 'read.native.enabled': 'true',
+ 'write.native.enabled': 'false',
+ }
+ data.update(extra)
+ return CoreOptions(Options(data))
+
+
+def _value_row(ts_type, fields=None):
+ if fields is not None:
+ return RowType(True, fields)
+ return RowType(True, [
+ DataField(2, 'value', AtomicType('STRING')),
+ DataField(3, 'event_time', AtomicType(ts_type)),
+ ])
+
+
+def _keytime_fields(value_type):
+ return [
+ DataField(0, 'id', AtomicType('STRING')),
+ DataField(1, 'updates', MapType(True, AtomicType('STRING'),
value_type)),
+ ]
+
+
+def _table(fields, options, primary_keys=('id',)):
+ table = Mock()
+ table.is_primary_key_table = bool(primary_keys)
+ table.primary_keys = list(primary_keys)
+ table.fields = fields
+ table.options = options
+ return table
+
+
+def _split():
+ split = Mock()
+ split.files = [Mock(
+ file_name='data-1.parquet', file_size=1, extra_files=[],
write_cols=None)]
+ split._native_split = object()
+ return split
+
+
+def _table_read(table):
+ from pypaimon.read.table_read import TableRead
+ from pypaimon.read.read_type import reader_adapter
+
+ read = TableRead.__new__(TableRead)
+ read.table = table
+ read.predicate = None
+ read.read_type = list(table.fields)
+ read.include_row_kind = False
+ read._row_kind_column = '_row_kind'
+ read.output_projection = None
+ read.limit = None
+ read._read_parallelism = 1
+ read._deferred_blob_fields = set()
+ read._predicate_extra_fields = []
+ read._adapter_read_type, _ = reader_adapter(read.read_type, table.fields)
+ read._scan_read_type = read._adapter_read_type
+ read._output_column_names = [field.name for field in read.read_type]
+ table.options.native_read_enabled = Mock(return_value=True)
+ table.options.file_format = Mock(return_value='parquet')
+ table.options.row_tracking_enabled = Mock(return_value=False)
+ table.options.data_evolution_enabled = Mock(return_value=False)
+ table.options.blob_as_descriptor = Mock(return_value=False)
+ table.options.blob_descriptor_fields = Mock(return_value=set())
+ table.options.blob_view_fields = Mock(return_value=set())
+ table.options.blob_view_resolve_enabled = Mock(return_value=True)
+ return read
+
+
+def _keytime_options(ts_field='event_time', agg='merge_map_with_keytime'):
+ options = {
+ 'fields.updates.aggregate-function': agg,
+ }
+ if ts_field is not None:
+ options['fields.updates.ts-field'] = ts_field
+ return _options(**options)
+
+
+def _assert_falls_back_before_native(table):
+ assert _native_merge_map_keytime_timestamp_supported(table) is False
+ read = _table_read(table)
+ with patch('pypaimon.execution.native.read.table_read.native_read') as
native:
+ assert NativeTableRead(read).read_batches(
+ [_split()], pa.schema([('id', pa.string())])) is None
+ native.assert_not_called()
+
+
+def _assert_stays_on_native(table, batch):
+ assert _native_merge_map_keytime_timestamp_supported(table) is True
+ read = _table_read(table)
+ with patch('pypaimon.execution.native.read.table_read.native_read',
+ return_value=[batch]) as native:
+ assert list(NativeTableRead(read).read_batches(
+ [_split()], batch.schema))
+ native.assert_called_once()
+
+
+# -- capability / early fallback (no Rust required) --------------------
+
+
[email protected]('ts_type', [
+ 'STRING', 'CHAR(8)', 'VARCHAR(16)',
+ 'BOOLEAN', 'BINARY(4)', 'VARBINARY(8)', 'BYTES',
+ 'TINYINT', 'SMALLINT', 'INT', 'INTEGER', 'BIGINT',
+ 'FLOAT', 'DOUBLE',
+ 'DATE', 'TIME', 'DECIMAL(10, 2)',
+ 'TIMESTAMP(9)', 'TIMESTAMP_LTZ(9)', 'TIMESTAMP',
+])
+def test_keytime_falls_back_before_opening_native(ts_type):
+ _assert_falls_back_before_native(_table(
+ _keytime_fields(_value_row(ts_type)),
+ _keytime_options()))
+
+
+def test_default_ts_field_also_falls_back():
+ double_last = _table(
+ _keytime_fields(_value_row('DOUBLE')),
+ _keytime_options(ts_field=None))
+ assert _native_merge_map_keytime_timestamp_supported(double_last) is False
+
+ string_last = _table(
+ _keytime_fields(RowType(True, [
+ DataField(2, 'event_time', AtomicType('DOUBLE')),
+ DataField(3, 'ts', AtomicType('STRING')),
+ ])),
+ _keytime_options(ts_field=None))
+ assert _native_merge_map_keytime_timestamp_supported(string_last) is False
+
+
+def test_non_pk_non_aggregate_and_other_aggs_stay_on_native():
+ fields = _keytime_fields(_value_row('STRING'))
+ cases = (
+ _table(fields, _keytime_options(), primary_keys=()),
+ _table(fields, _options(
+ **{'merge-engine': 'deduplicate',
+ 'fields.updates.aggregate-function': 'merge_map_with_keytime',
+ 'fields.updates.ts-field': 'event_time'})),
+ _table(fields, _options(
+ **{'fields.updates.aggregate-function': 'merge_map'})),
+ )
+ batch = pa.record_batch(
+ [pa.array(['1']), pa.array([None], type=pa.map_(
+ pa.string(), pa.struct([
+ ('value', pa.string()), ('event_time', pa.string())])))],
+ names=['id', 'updates'])
+ for table in cases:
+ _assert_stays_on_native(table, batch)
+
+ # Plain partial-update without keytime must keep Native.
+ pu = _table(
+ [DataField(0, 'id', AtomicType('STRING')),
+ DataField(1, 'v', AtomicType('STRING'))],
+ _options(**{'merge-engine': 'partial-update'}))
+ _assert_stays_on_native(
+ pu, pa.record_batch([pa.array(['1']), pa.array(['x'])], names=['id',
'v']))
+
+
+def _partial_update_keytime_fields():
+ return [
+ DataField(0, 'id', AtomicType('STRING')),
+ DataField(1, 'seq', AtomicType('INT')),
+ DataField(2, 'updates', MapType(True, AtomicType('STRING'),
_value_row('STRING'))),
+ ]
+
+
+def _partial_update_keytime_options():
+ return _options(**{
+ 'merge-engine': 'partial-update',
+ 'sequence.field': 'seq',
+ 'fields.seq.sequence-group': 'updates',
+ 'fields.updates.aggregate-function': 'merge_map_with_keytime',
+ 'fields.updates.ts-field': 'event_time',
+ })
+
+
+def test_partial_update_sequence_group_keytime_detected():
+ table = _table(_partial_update_keytime_fields(),
_partial_update_keytime_options())
+ assert _table_uses_merge_map_keytime(table) is True
+ assert _native_merge_map_keytime_timestamp_supported(table) is False
+ assert _native_write_refuses_merge_map_keytime(table) is True
+
+
+def test_partial_update_sequence_group_without_keytime_not_flagged():
+ table = _table(
+ _partial_update_keytime_fields(),
+ _options(**{
+ 'merge-engine': 'partial-update',
+ 'sequence.field': 'seq',
+ 'fields.seq.sequence-group': 'updates',
+ }))
+ assert _table_uses_merge_map_keytime(table) is False
+ assert _native_merge_map_keytime_timestamp_supported(table) is True
+ assert _native_write_refuses_merge_map_keytime(table) is False
+
+
+def test_partial_update_keytime_raises_before_opening_native():
+ table = _table(_partial_update_keytime_fields(),
_partial_update_keytime_options())
+ read = _table_read(table)
+ with patch('pypaimon.execution.native.read.table_read.native_read') as
native:
+ with pytest.raises(NotImplementedError, match='partial-update'):
+ NativeTableRead(read).read_batches(
+ [_split()], pa.schema([('id', pa.string())]))
+ native.assert_not_called()
+
+
+# -- real FileStore public-read fallback --------------------------------
+
+
+def _create_keytime_table(tmp_path, name, ts_pa_type, ts_field='event_time'):
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')})
+ catalog.create_database('default', True)
+ ts_name = 'ts' if ts_field is None else ts_field
+ value_type = pa.struct([
+ pa.field('value', pa.string()),
+ pa.field(ts_name, ts_pa_type),
+ ])
+ pa_schema = pa.schema([
+ pa.field('id', pa.string(), nullable=False),
+ pa.field('updates', pa.map_(pa.string(), value_type)),
+ ])
+ options = {
+ 'bucket': '1',
+ 'merge-engine': 'aggregation',
+ 'file.format': 'parquet',
+ 'fields.updates.aggregate-function': 'merge_map_with_keytime',
+ 'read.native.enabled': 'true',
+ 'write.native.enabled': 'false',
+ 'scan.native-plan.enabled': 'false',
+ }
+ if ts_field is not None:
+ options['fields.updates.ts-field'] = ts_name
+ schema = Schema.from_pyarrow_schema(
+ pa_schema, primary_keys=['id'], options=options)
+ catalog.create_table('default.{}'.format(name), schema, False)
+ table = catalog.get_table('default.{}'.format(name))
+ return table, pa_schema, ts_name
+
+
+def _write_two_versions(table, pa_schema, ts_name, older, newer):
+ for value, ts in (('old', older), ('new', newer)):
+ wb = table.new_batch_write_builder()
+ w = wb.new_write()
+ c = wb.new_commit()
+ try:
+ w.write_arrow(pa.Table.from_pylist([{
+ 'id': '1',
+ 'updates': [('k', {'value': value, ts_name: ts})],
+ }], schema=pa_schema))
+ c.commit(w.prepare_commit())
+ finally:
+ w.close()
+ c.close()
+
+
+def _public_read(table):
+ rb = table.new_read_builder()
+ splits = rb.new_scan().plan().splits()
+ return rb.new_read().to_arrow(splits)
+
+
+def
test_catalog_string_utf16_order_falls_back_and_keeps_python_winner(tmp_path):
+ table, pa_schema, ts_name = _create_keytime_table(
+ tmp_path, 'string_utf16', pa.string())
+ assert _native_merge_map_keytime_timestamp_supported(table) is False
+ older = '\uE000'
+ newer_under_utf8 = '\U00010000'
+ _write_two_versions(table, pa_schema, ts_name, older, newer_under_utf8)
+
+ read = table.new_read_builder().new_read()
+ splits = table.new_read_builder().new_scan().plan().splits()
+ schema = read._output_arrow_schema(splits)
+ with patch('pypaimon.execution.native.read.table_read.native_read') as
native:
+ assert NativeTableRead(read).read_batches(splits, schema) is None
+ native.assert_not_called()
+
+ rows = _public_read(table).to_pylist()
+ assert len(rows) == 1
+ updates = dict(rows[0]['updates'])
+ assert updates['k']['value'] == 'old'
+ assert updates['k'][ts_name] == older
+
+
+def test_catalog_ascii_string_still_prefers_lexically_newer(tmp_path):
+ table, pa_schema, ts_name = _create_keytime_table(
+ tmp_path, 'string_ascii', pa.string())
+ assert _native_merge_map_keytime_timestamp_supported(table) is False
+ _write_two_versions(table, pa_schema, ts_name, '100', '200')
+
+ read = table.new_read_builder().new_read()
+ splits = table.new_read_builder().new_scan().plan().splits()
+ schema = read._output_arrow_schema(splits)
+ with patch('pypaimon.execution.native.read.table_read.native_read') as
native:
+ assert NativeTableRead(read).read_batches(splits, schema) is None
+ native.assert_not_called()
+
+ rows = _public_read(table).to_pylist()
+ assert len(rows) == 1
+ updates = dict(rows[0]['updates'])
+ assert updates['k']['value'] == 'new'
+ assert updates['k'][ts_name] == '200'
+
+
+def test_catalog_double_falls_back_before_native(tmp_path):
+ table, pa_schema, ts_name = _create_keytime_table(
+ tmp_path, 'double_fallback', pa.float64())
+ assert _native_merge_map_keytime_timestamp_supported(table) is False
+ _write_two_versions(table, pa_schema, ts_name, 9.5, 10.25)
+
+ read = table.new_read_builder().new_read()
+ splits = table.new_read_builder().new_scan().plan().splits()
+ schema = read._output_arrow_schema(splits)
+ with patch('pypaimon.execution.native.read.table_read.native_read') as
native:
+ assert NativeTableRead(read).read_batches(splits, schema) is None
+ native.assert_not_called()
+
+ rows = _public_read(table).to_pylist()
+ assert len(rows) == 1
+ updates = dict(rows[0]['updates'])
+ assert updates['k']['value'] == 'new'
+ assert updates['k']['event_time'] == 10.25
+
+
+# -- Native Writer refuses keytime; Python keeps UTF-16 winner ------------
+
+
+def _create_keytime_table_native_write_requested(tmp_path, name):
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')})
+ catalog.create_database('default', True)
+ value_type = pa.struct([
+ pa.field('value', pa.string()),
+ pa.field('event_time', pa.string()),
+ ])
+ pa_schema = pa.schema([
+ pa.field('id', pa.string(), nullable=False),
+ pa.field('updates', pa.map_(pa.string(), value_type)),
+ ])
+ options = {
+ 'bucket': '1',
+ 'merge-engine': 'aggregation',
+ 'file.format': 'parquet',
+ 'fields.updates.aggregate-function': 'merge_map_with_keytime',
+ 'fields.updates.ts-field': 'event_time',
+ 'write.native.enabled': 'true',
+ 'read.native.enabled': 'true',
+ 'scan.native-plan.enabled': 'false',
+ 'commit.native.enabled': 'false',
+ }
+ schema = Schema.from_pyarrow_schema(
+ pa_schema, primary_keys=['id'], options=options)
+ catalog.create_table('default.{}'.format(name), schema, False)
+ return catalog.get_table('default.{}'.format(name)), pa_schema
+
+
+def test_native_writer_refuses_keytime_same_batch_keeps_utf16_first(tmp_path):
+ """Same write_arrow batch: Native refused; Python keeps UTF-16 first.
+
+ PMC scenario: one batch, same PK/map key, U+E000/first then
+ U+10000/second. Native UTF-8 would persist second; gated Python
+ Writer must keep first through commit and reopen.
+ """
+ table, pa_schema = _create_keytime_table_native_write_requested(
+ tmp_path, 'writer_utf16_gate')
+ assert _native_write_refuses_merge_map_keytime(table) is True
+
+ older = '\uE000'
+ newer_under_utf8 = '\U00010000'
+
+ with patch(
+
'pypaimon.execution.native.write.native_write.native_runtime_available',
+ return_value=True), \
+ patch(
+
'pypaimon.execution.native.write.native_write.create_native_write_table'
+ ) as create_table:
Review Comment:
[P2] Fix the indentation that blocks all Python CI test jobs
The closing parenthesis of this patch context triggers E123 with the
repository's dev/cfg.ini. The changed-file flake8 command reproduces it
locally, and CI43866912063 reports this exact line before any Python tests run
in all six Native/Legacy Python jobs. Please align this closing bracket with
its opening context and rerun the normal Python lint step so the regression
tests can execute in CI.
##########
paimon-python/pypaimon/execution/native/read/table_read.py:
##########
@@ -40,6 +40,75 @@
_NATIVE_READ_FILE_SUFFIXES = tuple('.%s' % file_format for file_format in
_NATIVE_READ_FILE_FORMATS)
_NATIVE_BLOB_FILE_SUFFIXES = ('.blob', '.video')
+# merge_map_with_keytime is unsafe on Native until Rust matches Java
+# UTF-16 string order (e.g. U+E000 vs U+10000):
+# * AGGREGATE: fall back to the Python aggregator before Native opens.
+# * PARTIAL_UPDATE (incl. sequence-group field aggregators): Python
+# cannot run this config either, so raise before Native opens rather
+# than returning a wrong UTF-8 winner.
+_MERGE_MAP_WITH_KEYTIME = "merge_map_with_keytime"
+
+
+def _field_agg_func_name(field_name, primary_keys, options_map,
sequence_fields):
+ # Same precedence as Python ``resolve_agg_func_name`` / Java
+ # ``AggregateMergeFunction.getAggFuncName``, inlined to avoid a
+ # Native → Python execution import.
+ if field_name in sequence_fields:
+ return "last_value"
+ if field_name in primary_keys:
+ return "primary_key"
+ return (
+ options_map.get("fields.{}.aggregate-function".format(field_name))
+ or options_map.get("fields.default-aggregate-function")
+ or "last_non_null_value"
+ )
+
+
+def _table_uses_merge_map_keytime(table) -> bool:
+ """True when a field aggregator actually resolves to keytime.
+
+ Covers Aggregate and Partial Update (including sequence-group tables
+ that set ``fields.<f>.aggregate-function=merge_map_with_keytime`` on
+ a group member). Detection follows field-aggregator precedence; it
+ does not treat ``sequence-group`` alone as keytime.
+ """
+ if not getattr(table, 'is_primary_key_table', False):
+ return False
+ options = table.options
+ if options.merge_engine() not in (
+ MergeEngine.AGGREGATE, MergeEngine.PARTIAL_UPDATE):
+ return False
+
+ options_map = options.options.to_map()
+ pk_set = set(table.primary_keys)
+ sequence_fields = set(options.sequence_field())
Review Comment:
[P2] Exclude sequence-group ordering fields from effective aggregation
For partial-update, Java resolves field aggregators using allSequenceFields,
including the ordering fields from fields.<seq>.sequence-group. This helper
only excludes the top-level sequence.field list. A valid table with PK=id,
fields.seq.sequence-group=updates,
fields.default-aggregate-function=merge_map_with_keytime and
fields.updates.aggregate-function=last_non_null_value has no effective keytime
aggregator: seq is an ordering field and updates overrides the default. The new
read and mirrored write gates nevertheless classify seq as keytime and reject
both public operations with NotImplementedError. Using the same real Rust 0.5
runtime and persisted files, bypassing only these newly added gates restores
successful Native write/commit and the expected seq=2, updates=second read.
Please include sequence-group ordering fields when resolving effective
aggregators in both gates and add read/write regressions for this configuration.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]