JingsongLi commented on code in PR #1378:
URL: https://github.com/apache/paimon-rust/pull/1378#discussion_r4581705862
##########
crates/paimon/src/table/aggregator/merge_map.rs:
##########
@@ -396,11 +407,228 @@ impl FieldAggregator for MergeMapAgg {
}
}
+/// Schema/runtime type check shared with [`MergeMapAgg::new_with_keytime`].
+pub(crate) fn validate_merge_map_with_keytime(
+ field_name: &str,
+ data_type: &DataType,
+ options: &HashMap<String, String>,
+) -> crate::Result<()> {
+ resolve_keytime_config(field_name, data_type, options).map(|_| ())
+}
+
+fn resolve_keytime_config(
+ field_name: &str,
+ data_type: &DataType,
+ options: &HashMap<String, String>,
+) -> crate::Result<(usize, KeytimeKind)> {
+ let DataType::Map(map_type) = data_type else {
+ return Err(unsupported_type_error(
+ "merge_map_with_keytime",
+ field_name,
+ data_type,
+ ));
+ };
+ let DataType::Row(value_type) = map_type.value_type() else {
+ return Err(unsupported_type_error(
+ "merge_map_with_keytime",
+ field_name,
+ data_type,
+ ));
+ };
+ if value_type.fields().len() < 2 {
+ return Err(unsupported_type_error(
+ "merge_map_with_keytime",
+ field_name,
+ data_type,
+ ));
+ }
+ let timestamp_index = match
options.get(&format!("fields.{field_name}.ts-field")) {
+ Some(name) => value_type
+ .fields()
+ .iter()
+ .position(|field| field.name() == name)
+ .ok_or_else(|| Error::ConfigInvalid {
+ message: format!(
+ "Timestamp field '{name}' not found in ROW type for field
'{field_name}'"
+ ),
+ })?,
+ None => value_type.fields().len() - 1,
+ };
+ let ts_field = &value_type.fields()[timestamp_index];
+ let keytime_kind = keytime_kind_for_type(field_name, ts_field.name(),
ts_field.data_type())?;
+ Ok((timestamp_index, keytime_kind))
+}
+
+fn keytime_kind_for_type(
+ field_name: &str,
+ ts_field_name: &str,
+ ts_type: &DataType,
+) -> crate::Result<KeytimeKind> {
+ match ts_type {
+ DataType::Char(_) | DataType::VarChar(_) => Ok(KeytimeKind::Utf8),
+ DataType::TinyInt(_) => Ok(KeytimeKind::Int8),
+ DataType::SmallInt(_) => Ok(KeytimeKind::Int16),
+ DataType::Int(_) => Ok(KeytimeKind::Int32),
+ DataType::BigInt(_) => Ok(KeytimeKind::Int64),
+ DataType::Float(_) => Ok(KeytimeKind::Float32),
+ DataType::Double(_) => Ok(KeytimeKind::Float64),
+ DataType::Date(_) => Ok(KeytimeKind::Date32),
+ DataType::Time(_) => Ok(KeytimeKind::Time32Ms),
+ DataType::Decimal(_) => Ok(KeytimeKind::Decimal128),
+ DataType::Timestamp(_) | DataType::LocalZonedTimestamp(_) => {
+ match paimon_type_to_arrow(ts_type)? {
+ ArrowDataType::Timestamp(unit, _) => Ok(KeytimeKind::Timestamp
{ unit }),
+ other => Err(Error::DataInvalid {
+ message: format!(
+ "Timestamp field '{ts_field_name}' for field
'{field_name}' \
+ maps to unexpected Arrow type {other:?}"
+ ),
+ source: None,
+ }),
+ }
+ }
+ other => Err(Error::ConfigInvalid {
+ message: format!(
+ "Timestamp field '{ts_field_name}' for field '{field_name}'
must be \
+ CHAR/VARCHAR, TINYINT/SMALLINT/INT/BIGINT, FLOAT/DOUBLE,
DATE, TIME, \
+ DECIMAL, TIMESTAMP or TIMESTAMP_LTZ, but was {other:?}."
+ ),
+ }),
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum KeytimeCmp {
+ Less,
+ Equal,
+ Greater,
+ /// Python float ordering: any comparison involving NaN is unordered, so
+ /// the incoming value must not replace the existing one.
+ Unordered,
+}
+
+fn compare_keytime(
+ kind: &KeytimeKind,
+ new_ts: &dyn Array,
+ old_ts: &dyn Array,
+ field_name: &str,
+) -> crate::Result<KeytimeCmp> {
+ match kind {
+ KeytimeKind::Utf8 => {
+ let new_v = downcast::<StringArray>(new_ts, field_name)?.value(0);
+ let old_v = downcast::<StringArray>(old_ts, field_name)?.value(0);
+ Ok(ordering_to_cmp(utf16_code_unit_cmp(new_v, old_v)))
+ }
+ KeytimeKind::Int8 => cmp_ord(
+ downcast::<Int8Array>(new_ts, field_name)?.value(0),
+ downcast::<Int8Array>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Int16 => cmp_ord(
+ downcast::<Int16Array>(new_ts, field_name)?.value(0),
+ downcast::<Int16Array>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Int32 => cmp_ord(
+ downcast::<Int32Array>(new_ts, field_name)?.value(0),
+ downcast::<Int32Array>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Int64 => cmp_ord(
+ downcast::<Int64Array>(new_ts, field_name)?.value(0),
+ downcast::<Int64Array>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Float32 => Ok(python_float_cmp(
+ downcast::<Float32Array>(new_ts, field_name)?.value(0),
+ downcast::<Float32Array>(old_ts, field_name)?.value(0),
+ )),
+ KeytimeKind::Float64 => Ok(python_float_cmp(
+ downcast::<Float64Array>(new_ts, field_name)?.value(0),
+ downcast::<Float64Array>(old_ts, field_name)?.value(0),
+ )),
+ KeytimeKind::Date32 => cmp_ord(
+ downcast::<Date32Array>(new_ts, field_name)?.value(0),
+ downcast::<Date32Array>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Time32Ms => cmp_ord(
+ downcast::<Time32MillisecondArray>(new_ts, field_name)?.value(0),
+ downcast::<Time32MillisecondArray>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Decimal128 => cmp_ord(
+ downcast::<Decimal128Array>(new_ts, field_name)?.value(0),
+ downcast::<Decimal128Array>(old_ts, field_name)?.value(0),
+ ),
+ KeytimeKind::Timestamp { unit } => {
+ let new_v = timestamp_value(new_ts, *unit, field_name)?;
+ let old_v = timestamp_value(old_ts, *unit, field_name)?;
+ cmp_ord(new_v, old_v)
+ }
+ }
+}
+
+fn cmp_ord<T: Ord>(new_v: T, old_v: T) -> crate::Result<KeytimeCmp> {
+ Ok(ordering_to_cmp(new_v.cmp(&old_v)))
+}
+
+fn ordering_to_cmp(ordering: Ordering) -> KeytimeCmp {
+ match ordering {
+ Ordering::Less => KeytimeCmp::Less,
+ Ordering::Equal => KeytimeCmp::Equal,
+ Ordering::Greater => KeytimeCmp::Greater,
+ }
+}
+
+/// Python `>` / `==` for keytime floats: NaN never updates; ±0.0 compare
equal.
+fn python_float_cmp<T: Copy + PartialEq + PartialOrd>(new_v: T, old_v: T) ->
KeytimeCmp {
+ if new_v > old_v {
+ KeytimeCmp::Greater
+ } else if new_v == old_v {
+ KeytimeCmp::Equal
+ } else if new_v < old_v {
+ KeytimeCmp::Less
+ } else {
+ KeytimeCmp::Unordered
Review Comment:
[P1] Make floating key-time reduction independent of flush grouping
Treating NaN as unordered against every value makes this reducer
non-associative. I reproduced it with a real aggregation table, one map key,
and arrival rows (payload=11,ts=0), (22,NaN), (33,1). Three separate writer
flush/commits read back 33 and retain 33 after compaction; committing the first
row separately but buffering the last two together reads back 11. The latter
buffer first reduces NaN and 1 to NaN, losing the update that would have won
against 0. This silently changes persisted results based only on batching for
newly accepted DOUBLE/FLOAT key-times. Define a merge-safe NaN policy or reject
NaNs before accepting this type; matching Python's two-value comparison is
insufficient for LSM partial aggregation.
##########
crates/paimon/src/table/aggregator/merge_map.rs:
##########
@@ -72,41 +100,15 @@ impl MergeMapAgg {
data_type: &DataType,
options: &HashMap<String, String>,
) -> crate::Result<Self> {
- let DataType::Map(map_type) = data_type else {
- return Err(unsupported_type_error(
- "merge_map_with_keytime",
- field_name,
- data_type,
- ));
- };
- let DataType::Row(value_type) = map_type.value_type() else {
- return Err(unsupported_type_error(
- "merge_map_with_keytime",
- field_name,
- data_type,
- ));
- };
- if value_type.fields().len() < 2 {
- return Err(unsupported_type_error(
- "merge_map_with_keytime",
- field_name,
- data_type,
- ));
- }
- let timestamp_index = match
options.get(&format!("fields.{field_name}.ts-field")) {
- Some(name) => value_type
- .fields()
- .iter()
- .position(|field| field.name() == name)
- .ok_or_else(|| Error::ConfigInvalid {
- message: format!(
- "Timestamp field '{name}' not found in ROW type for
field '{field_name}'"
- ),
- })?,
- None => value_type.fields().len() - 1,
- };
+ let (timestamp_index, keytime_kind) =
+ resolve_keytime_config(field_name, data_type, options)?;
+ let overwrite_on_equal_ts = options
+ .get(&format!("fields.{field_name}.overwrite-on-equal-ts"))
Review Comment:
[P2] Register the new field option with schema evolution
The new suffix is absent from parse_field_scoped_option_key and
FIELD_SCOPED_RENAMEABLE_SUFFIXES. A valid table with
fields.m.aggregate-function=merge_map_with_keytime and
fields.m.overwrite-on-equal-ts=true (using the default last ts field) can
successfully rename m to renamed. apply_changes moves the aggregate-function
but leaves this option under fields.m, so the renamed aggregator silently
reverts to false and selects a different winner for equal timestamps. Dropping
m also leaves the option behind, allowing it to reactivate if the name is
reused. Both public TableSchema::apply_changes probes fail on this head.
Include the option in recognition/rename/drop handling and cover its policy
through schema evolution.
##########
crates/paimon/src/table/aggregator/merge_map.rs:
##########
@@ -396,11 +407,228 @@ impl FieldAggregator for MergeMapAgg {
}
}
+/// Schema/runtime type check shared with [`MergeMapAgg::new_with_keytime`].
+pub(crate) fn validate_merge_map_with_keytime(
+ field_name: &str,
+ data_type: &DataType,
+ options: &HashMap<String, String>,
+) -> crate::Result<()> {
+ resolve_keytime_config(field_name, data_type, options).map(|_| ())
+}
+
+fn resolve_keytime_config(
+ field_name: &str,
+ data_type: &DataType,
+ options: &HashMap<String, String>,
+) -> crate::Result<(usize, KeytimeKind)> {
+ let DataType::Map(map_type) = data_type else {
+ return Err(unsupported_type_error(
+ "merge_map_with_keytime",
+ field_name,
+ data_type,
+ ));
+ };
+ let DataType::Row(value_type) = map_type.value_type() else {
+ return Err(unsupported_type_error(
+ "merge_map_with_keytime",
+ field_name,
+ data_type,
+ ));
+ };
+ if value_type.fields().len() < 2 {
+ return Err(unsupported_type_error(
+ "merge_map_with_keytime",
+ field_name,
+ data_type,
+ ));
+ }
+ let timestamp_index = match
options.get(&format!("fields.{field_name}.ts-field")) {
+ Some(name) => value_type
+ .fields()
+ .iter()
+ .position(|field| field.name() == name)
+ .ok_or_else(|| Error::ConfigInvalid {
+ message: format!(
+ "Timestamp field '{name}' not found in ROW type for field
'{field_name}'"
+ ),
+ })?,
+ None => value_type.fields().len() - 1,
+ };
+ let ts_field = &value_type.fields()[timestamp_index];
+ let keytime_kind = keytime_kind_for_type(field_name, ts_field.name(),
ts_field.data_type())?;
+ Ok((timestamp_index, keytime_kind))
+}
+
+fn keytime_kind_for_type(
+ field_name: &str,
+ ts_field_name: &str,
+ ts_type: &DataType,
+) -> crate::Result<KeytimeKind> {
+ match ts_type {
+ DataType::Char(_) | DataType::VarChar(_) => Ok(KeytimeKind::Utf8),
+ DataType::TinyInt(_) => Ok(KeytimeKind::Int8),
+ DataType::SmallInt(_) => Ok(KeytimeKind::Int16),
+ DataType::Int(_) => Ok(KeytimeKind::Int32),
+ DataType::BigInt(_) => Ok(KeytimeKind::Int64),
+ DataType::Float(_) => Ok(KeytimeKind::Float32),
+ DataType::Double(_) => Ok(KeytimeKind::Float64),
+ DataType::Date(_) => Ok(KeytimeKind::Date32),
+ DataType::Time(_) => Ok(KeytimeKind::Time32Ms),
+ DataType::Decimal(_) => Ok(KeytimeKind::Decimal128),
+ DataType::Timestamp(_) | DataType::LocalZonedTimestamp(_) => {
+ match paimon_type_to_arrow(ts_type)? {
+ ArrowDataType::Timestamp(unit, _) => Ok(KeytimeKind::Timestamp
{ unit }),
+ other => Err(Error::DataInvalid {
+ message: format!(
+ "Timestamp field '{ts_field_name}' for field
'{field_name}' \
+ maps to unexpected Arrow type {other:?}"
+ ),
+ source: None,
+ }),
+ }
+ }
+ other => Err(Error::ConfigInvalid {
+ message: format!(
+ "Timestamp field '{ts_field_name}' for field '{field_name}'
must be \
+ CHAR/VARCHAR, TINYINT/SMALLINT/INT/BIGINT, FLOAT/DOUBLE,
DATE, TIME, \
+ DECIMAL, TIMESTAMP or TIMESTAMP_LTZ, but was {other:?}."
+ ),
+ }),
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum KeytimeCmp {
+ Less,
+ Equal,
+ Greater,
+ /// Python float ordering: any comparison involving NaN is unordered, so
+ /// the incoming value must not replace the existing one.
+ Unordered,
+}
+
+fn compare_keytime(
+ kind: &KeytimeKind,
+ new_ts: &dyn Array,
+ old_ts: &dyn Array,
+ field_name: &str,
+) -> crate::Result<KeytimeCmp> {
+ match kind {
+ KeytimeKind::Utf8 => {
+ let new_v = downcast::<StringArray>(new_ts, field_name)?.value(0);
+ let old_v = downcast::<StringArray>(old_ts, field_name)?.value(0);
+ Ok(ordering_to_cmp(utf16_code_unit_cmp(new_v, old_v)))
Review Comment:
[P2] Preserve BinaryString ordering for existing character key-times
Java's key-time aggregator calls InternalRowUtils.compare, whose
CHAR/VARCHAR branch compares BinaryString values, not java.lang.String values.
BinaryString.compareTo orders unsigned UTF-8 bytes. Changing the previous Rust
byte order to UTF-16 reverses supplementary-character comparisons and chooses a
different row on existing tables. Public write/read probe: first (payload=11,
ts=U+E000), then (22,U+10000); this head returns 11, while the exact base
returns 22 and Java's BinaryString ordering also selects 22. Keep the existing
UTF-8 comparison and correct the new test/docs oracle. Reference:
https://github.com/apache/paimon/blob/f289b0a2829f8b0c28ae92743f11893893ce1116/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java#L487-L489
and
https://github.com/apache/paimon/blob/f289b0a2829f8b0c28ae92743f11893893ce1116/paimon-common/src/main/java/org/apache/paimon/data/BinaryString.java#L103-L126
.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]