JingsongLi commented on code in PR #1378:
URL: https://github.com/apache/paimon-rust/pull/1378#discussion_r4581705862


##########
crates/paimon/src/table/aggregator/merge_map.rs:
##########
@@ -396,11 +407,228 @@ impl FieldAggregator for MergeMapAgg {
     }
 }
 
+/// Schema/runtime type check shared with [`MergeMapAgg::new_with_keytime`].
+pub(crate) fn validate_merge_map_with_keytime(
+    field_name: &str,
+    data_type: &DataType,
+    options: &HashMap<String, String>,
+) -> crate::Result<()> {
+    resolve_keytime_config(field_name, data_type, options).map(|_| ())
+}
+
+fn resolve_keytime_config(
+    field_name: &str,
+    data_type: &DataType,
+    options: &HashMap<String, String>,
+) -> crate::Result<(usize, KeytimeKind)> {
+    let DataType::Map(map_type) = data_type else {
+        return Err(unsupported_type_error(
+            "merge_map_with_keytime",
+            field_name,
+            data_type,
+        ));
+    };
+    let DataType::Row(value_type) = map_type.value_type() else {
+        return Err(unsupported_type_error(
+            "merge_map_with_keytime",
+            field_name,
+            data_type,
+        ));
+    };
+    if value_type.fields().len() < 2 {
+        return Err(unsupported_type_error(
+            "merge_map_with_keytime",
+            field_name,
+            data_type,
+        ));
+    }
+    let timestamp_index = match 
options.get(&format!("fields.{field_name}.ts-field")) {
+        Some(name) => value_type
+            .fields()
+            .iter()
+            .position(|field| field.name() == name)
+            .ok_or_else(|| Error::ConfigInvalid {
+                message: format!(
+                    "Timestamp field '{name}' not found in ROW type for field 
'{field_name}'"
+                ),
+            })?,
+        None => value_type.fields().len() - 1,
+    };
+    let ts_field = &value_type.fields()[timestamp_index];
+    let keytime_kind = keytime_kind_for_type(field_name, ts_field.name(), 
ts_field.data_type())?;
+    Ok((timestamp_index, keytime_kind))
+}
+
+fn keytime_kind_for_type(
+    field_name: &str,
+    ts_field_name: &str,
+    ts_type: &DataType,
+) -> crate::Result<KeytimeKind> {
+    match ts_type {
+        DataType::Char(_) | DataType::VarChar(_) => Ok(KeytimeKind::Utf8),
+        DataType::TinyInt(_) => Ok(KeytimeKind::Int8),
+        DataType::SmallInt(_) => Ok(KeytimeKind::Int16),
+        DataType::Int(_) => Ok(KeytimeKind::Int32),
+        DataType::BigInt(_) => Ok(KeytimeKind::Int64),
+        DataType::Float(_) => Ok(KeytimeKind::Float32),
+        DataType::Double(_) => Ok(KeytimeKind::Float64),
+        DataType::Date(_) => Ok(KeytimeKind::Date32),
+        DataType::Time(_) => Ok(KeytimeKind::Time32Ms),
+        DataType::Decimal(_) => Ok(KeytimeKind::Decimal128),
+        DataType::Timestamp(_) | DataType::LocalZonedTimestamp(_) => {
+            match paimon_type_to_arrow(ts_type)? {
+                ArrowDataType::Timestamp(unit, _) => Ok(KeytimeKind::Timestamp 
{ unit }),
+                other => Err(Error::DataInvalid {
+                    message: format!(
+                        "Timestamp field '{ts_field_name}' for field 
'{field_name}' \
+                         maps to unexpected Arrow type {other:?}"
+                    ),
+                    source: None,
+                }),
+            }
+        }
+        other => Err(Error::ConfigInvalid {
+            message: format!(
+                "Timestamp field '{ts_field_name}' for field '{field_name}' 
must be \
+                 CHAR/VARCHAR, TINYINT/SMALLINT/INT/BIGINT, FLOAT/DOUBLE, 
DATE, TIME, \
+                 DECIMAL, TIMESTAMP or TIMESTAMP_LTZ, but was {other:?}."
+            ),
+        }),
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum KeytimeCmp {
+    Less,
+    Equal,
+    Greater,
+    /// Python float ordering: any comparison involving NaN is unordered, so
+    /// the incoming value must not replace the existing one.
+    Unordered,
+}
+
+fn compare_keytime(
+    kind: &KeytimeKind,
+    new_ts: &dyn Array,
+    old_ts: &dyn Array,
+    field_name: &str,
+) -> crate::Result<KeytimeCmp> {
+    match kind {
+        KeytimeKind::Utf8 => {
+            let new_v = downcast::<StringArray>(new_ts, field_name)?.value(0);
+            let old_v = downcast::<StringArray>(old_ts, field_name)?.value(0);
+            Ok(ordering_to_cmp(utf16_code_unit_cmp(new_v, old_v)))
+        }
+        KeytimeKind::Int8 => cmp_ord(
+            downcast::<Int8Array>(new_ts, field_name)?.value(0),
+            downcast::<Int8Array>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Int16 => cmp_ord(
+            downcast::<Int16Array>(new_ts, field_name)?.value(0),
+            downcast::<Int16Array>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Int32 => cmp_ord(
+            downcast::<Int32Array>(new_ts, field_name)?.value(0),
+            downcast::<Int32Array>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Int64 => cmp_ord(
+            downcast::<Int64Array>(new_ts, field_name)?.value(0),
+            downcast::<Int64Array>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Float32 => Ok(python_float_cmp(
+            downcast::<Float32Array>(new_ts, field_name)?.value(0),
+            downcast::<Float32Array>(old_ts, field_name)?.value(0),
+        )),
+        KeytimeKind::Float64 => Ok(python_float_cmp(
+            downcast::<Float64Array>(new_ts, field_name)?.value(0),
+            downcast::<Float64Array>(old_ts, field_name)?.value(0),
+        )),
+        KeytimeKind::Date32 => cmp_ord(
+            downcast::<Date32Array>(new_ts, field_name)?.value(0),
+            downcast::<Date32Array>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Time32Ms => cmp_ord(
+            downcast::<Time32MillisecondArray>(new_ts, field_name)?.value(0),
+            downcast::<Time32MillisecondArray>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Decimal128 => cmp_ord(
+            downcast::<Decimal128Array>(new_ts, field_name)?.value(0),
+            downcast::<Decimal128Array>(old_ts, field_name)?.value(0),
+        ),
+        KeytimeKind::Timestamp { unit } => {
+            let new_v = timestamp_value(new_ts, *unit, field_name)?;
+            let old_v = timestamp_value(old_ts, *unit, field_name)?;
+            cmp_ord(new_v, old_v)
+        }
+    }
+}
+
+fn cmp_ord<T: Ord>(new_v: T, old_v: T) -> crate::Result<KeytimeCmp> {
+    Ok(ordering_to_cmp(new_v.cmp(&old_v)))
+}
+
+fn ordering_to_cmp(ordering: Ordering) -> KeytimeCmp {
+    match ordering {
+        Ordering::Less => KeytimeCmp::Less,
+        Ordering::Equal => KeytimeCmp::Equal,
+        Ordering::Greater => KeytimeCmp::Greater,
+    }
+}
+
+/// Python `>` / `==` for keytime floats: NaN never updates; ±0.0 compare 
equal.
+fn python_float_cmp<T: Copy + PartialEq + PartialOrd>(new_v: T, old_v: T) -> 
KeytimeCmp {
+    if new_v > old_v {
+        KeytimeCmp::Greater
+    } else if new_v == old_v {
+        KeytimeCmp::Equal
+    } else if new_v < old_v {
+        KeytimeCmp::Less
+    } else {
+        KeytimeCmp::Unordered

Review Comment:
   [P1] Make floating key-time reduction independent of flush grouping
   
   Treating NaN as unordered against every value makes this reducer 
non-associative. I reproduced it with a real aggregation table, one map key, 
and arrival rows (payload=11,ts=0), (22,NaN), (33,1). Three separate writer 
flush/commits read back 33 and retain 33 after compaction; committing the first 
row separately but buffering the last two together reads back 11. The latter 
buffer first reduces NaN and 1 to NaN, losing the update that would have won 
against 0. This silently changes persisted results based only on batching for 
newly accepted DOUBLE/FLOAT key-times. Define a merge-safe NaN policy or reject 
NaNs before accepting this type; matching Python's two-value comparison is 
insufficient for LSM partial aggregation.



##########
crates/paimon/src/table/aggregator/merge_map.rs:
##########
@@ -72,41 +100,15 @@ impl MergeMapAgg {
         data_type: &DataType,
         options: &HashMap<String, String>,
     ) -> crate::Result<Self> {
-        let DataType::Map(map_type) = data_type else {
-            return Err(unsupported_type_error(
-                "merge_map_with_keytime",
-                field_name,
-                data_type,
-            ));
-        };
-        let DataType::Row(value_type) = map_type.value_type() else {
-            return Err(unsupported_type_error(
-                "merge_map_with_keytime",
-                field_name,
-                data_type,
-            ));
-        };
-        if value_type.fields().len() < 2 {
-            return Err(unsupported_type_error(
-                "merge_map_with_keytime",
-                field_name,
-                data_type,
-            ));
-        }
-        let timestamp_index = match 
options.get(&format!("fields.{field_name}.ts-field")) {
-            Some(name) => value_type
-                .fields()
-                .iter()
-                .position(|field| field.name() == name)
-                .ok_or_else(|| Error::ConfigInvalid {
-                    message: format!(
-                        "Timestamp field '{name}' not found in ROW type for 
field '{field_name}'"
-                    ),
-                })?,
-            None => value_type.fields().len() - 1,
-        };
+        let (timestamp_index, keytime_kind) =
+            resolve_keytime_config(field_name, data_type, options)?;
+        let overwrite_on_equal_ts = options
+            .get(&format!("fields.{field_name}.overwrite-on-equal-ts"))

Review Comment:
   [P2] Register the new field option with schema evolution
   
   The new suffix is absent from parse_field_scoped_option_key and 
FIELD_SCOPED_RENAMEABLE_SUFFIXES. A valid table with 
fields.m.aggregate-function=merge_map_with_keytime and 
fields.m.overwrite-on-equal-ts=true (using the default last ts field) can 
successfully rename m to renamed. apply_changes moves the aggregate-function 
but leaves this option under fields.m, so the renamed aggregator silently 
reverts to false and selects a different winner for equal timestamps. Dropping 
m also leaves the option behind, allowing it to reactivate if the name is 
reused. Both public TableSchema::apply_changes probes fail on this head. 
Include the option in recognition/rename/drop handling and cover its policy 
through schema evolution.



##########
crates/paimon/src/table/aggregator/merge_map.rs:
##########
@@ -396,11 +407,228 @@ impl FieldAggregator for MergeMapAgg {
     }
 }
 
+/// Schema/runtime type check shared with [`MergeMapAgg::new_with_keytime`].
+pub(crate) fn validate_merge_map_with_keytime(
+    field_name: &str,
+    data_type: &DataType,
+    options: &HashMap<String, String>,
+) -> crate::Result<()> {
+    resolve_keytime_config(field_name, data_type, options).map(|_| ())
+}
+
+fn resolve_keytime_config(
+    field_name: &str,
+    data_type: &DataType,
+    options: &HashMap<String, String>,
+) -> crate::Result<(usize, KeytimeKind)> {
+    let DataType::Map(map_type) = data_type else {
+        return Err(unsupported_type_error(
+            "merge_map_with_keytime",
+            field_name,
+            data_type,
+        ));
+    };
+    let DataType::Row(value_type) = map_type.value_type() else {
+        return Err(unsupported_type_error(
+            "merge_map_with_keytime",
+            field_name,
+            data_type,
+        ));
+    };
+    if value_type.fields().len() < 2 {
+        return Err(unsupported_type_error(
+            "merge_map_with_keytime",
+            field_name,
+            data_type,
+        ));
+    }
+    let timestamp_index = match 
options.get(&format!("fields.{field_name}.ts-field")) {
+        Some(name) => value_type
+            .fields()
+            .iter()
+            .position(|field| field.name() == name)
+            .ok_or_else(|| Error::ConfigInvalid {
+                message: format!(
+                    "Timestamp field '{name}' not found in ROW type for field 
'{field_name}'"
+                ),
+            })?,
+        None => value_type.fields().len() - 1,
+    };
+    let ts_field = &value_type.fields()[timestamp_index];
+    let keytime_kind = keytime_kind_for_type(field_name, ts_field.name(), 
ts_field.data_type())?;
+    Ok((timestamp_index, keytime_kind))
+}
+
+fn keytime_kind_for_type(
+    field_name: &str,
+    ts_field_name: &str,
+    ts_type: &DataType,
+) -> crate::Result<KeytimeKind> {
+    match ts_type {
+        DataType::Char(_) | DataType::VarChar(_) => Ok(KeytimeKind::Utf8),
+        DataType::TinyInt(_) => Ok(KeytimeKind::Int8),
+        DataType::SmallInt(_) => Ok(KeytimeKind::Int16),
+        DataType::Int(_) => Ok(KeytimeKind::Int32),
+        DataType::BigInt(_) => Ok(KeytimeKind::Int64),
+        DataType::Float(_) => Ok(KeytimeKind::Float32),
+        DataType::Double(_) => Ok(KeytimeKind::Float64),
+        DataType::Date(_) => Ok(KeytimeKind::Date32),
+        DataType::Time(_) => Ok(KeytimeKind::Time32Ms),
+        DataType::Decimal(_) => Ok(KeytimeKind::Decimal128),
+        DataType::Timestamp(_) | DataType::LocalZonedTimestamp(_) => {
+            match paimon_type_to_arrow(ts_type)? {
+                ArrowDataType::Timestamp(unit, _) => Ok(KeytimeKind::Timestamp 
{ unit }),
+                other => Err(Error::DataInvalid {
+                    message: format!(
+                        "Timestamp field '{ts_field_name}' for field 
'{field_name}' \
+                         maps to unexpected Arrow type {other:?}"
+                    ),
+                    source: None,
+                }),
+            }
+        }
+        other => Err(Error::ConfigInvalid {
+            message: format!(
+                "Timestamp field '{ts_field_name}' for field '{field_name}' 
must be \
+                 CHAR/VARCHAR, TINYINT/SMALLINT/INT/BIGINT, FLOAT/DOUBLE, 
DATE, TIME, \
+                 DECIMAL, TIMESTAMP or TIMESTAMP_LTZ, but was {other:?}."
+            ),
+        }),
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum KeytimeCmp {
+    Less,
+    Equal,
+    Greater,
+    /// Python float ordering: any comparison involving NaN is unordered, so
+    /// the incoming value must not replace the existing one.
+    Unordered,
+}
+
+fn compare_keytime(
+    kind: &KeytimeKind,
+    new_ts: &dyn Array,
+    old_ts: &dyn Array,
+    field_name: &str,
+) -> crate::Result<KeytimeCmp> {
+    match kind {
+        KeytimeKind::Utf8 => {
+            let new_v = downcast::<StringArray>(new_ts, field_name)?.value(0);
+            let old_v = downcast::<StringArray>(old_ts, field_name)?.value(0);
+            Ok(ordering_to_cmp(utf16_code_unit_cmp(new_v, old_v)))

Review Comment:
   [P2] Preserve BinaryString ordering for existing character key-times
   
   Java's key-time aggregator calls InternalRowUtils.compare, whose 
CHAR/VARCHAR branch compares BinaryString values, not java.lang.String values. 
BinaryString.compareTo orders unsigned UTF-8 bytes. Changing the previous Rust 
byte order to UTF-16 reverses supplementary-character comparisons and chooses a 
different row on existing tables. Public write/read probe: first (payload=11, 
ts=U+E000), then (22,U+10000); this head returns 11, while the exact base 
returns 22 and Java's BinaryString ordering also selects 22. Keep the existing 
UTF-8 comparison and correct the new test/docs oracle. Reference: 
https://github.com/apache/paimon/blob/f289b0a2829f8b0c28ae92743f11893893ce1116/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java#L487-L489
 and 
https://github.com/apache/paimon/blob/f289b0a2829f8b0c28ae92743f11893893ce1116/paimon-common/src/main/java/org/apache/paimon/data/BinaryString.java#L103-L126
 .



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to