LuciferYang commented on code in PR #1352:
URL: https://github.com/apache/paimon-rust/pull/1352#discussion_r4521047368
##########
crates/paimon/src/table/aggregator/collect.rs:
##########
@@ -34,119 +37,252 @@ pub(crate) struct CollectAgg {
field_name: String,
element: FieldRef,
distinct: bool,
+ /// A positive limit keeps the first N elements in merge order, a
+ /// negative one keeps the last |N|.
+ count_limit: i32,
java_uses_equaliser: bool,
+ /// Row encoder for distinct elements that Java collects into a
+ /// `LinkedHashSet`. `None` for other columns, and for element types
+ /// arrow-row cannot encode, which keep the linear comparison.
+ key_encoder: Option<RowConverter>,
seen_input: bool,
+ /// The elements are a DELETE value kept as-is, so they may hold
+ /// duplicates or exceed the limit until the next merge.
raw_accumulator: bool,
elements: Vec<ArrayRef>,
+ /// Encoded keys of `elements` when `key_encoder` is set; `None` until
+ /// rebuilt after a retract.
+ element_keys: Option<HashSet<Box<[u8]>>>,
+}
+
+/// One side of Java's merge, with the encoded keys of its elements when
+/// the column deduplicates through a set.
+struct Operand {
+ elements: Vec<ArrayRef>,
+ keys: Option<Rows>,
+ has_duplicate: bool,
}
impl CollectAgg {
pub(crate) fn new(
field_name: &str,
data_type: &DataType,
distinct: bool,
+ count_limit: i32,
) -> crate::Result<Self> {
- if !matches!(data_type, DataType::Array(_)) {
+ let DataType::Array(array_type) = data_type else {
return Err(unsupported_type_error("collect", field_name,
data_type));
- }
+ };
let ArrowDataType::List(element) = paimon_type_to_arrow(data_type)?
else {
unreachable!("ARRAY must map to Arrow List")
};
+ // Java `FieldCollectAgg#needsEqualiser`: constructed types and
+ // BINARY/VARBINARY (Rust has no GEOMETRY/GEOGRAPHY).
+ let java_uses_equaliser = matches!(
+ array_type.element_type(),
+ DataType::Array(_)
+ | DataType::Vector(_)
+ | DataType::Multiset(_)
+ | DataType::Map(_)
+ | DataType::Row(_)
+ | DataType::Binary(_)
+ | DataType::VarBinary(_)
+ );
+ let key_encoder = if distinct && !java_uses_equaliser {
+
RowConverter::new(vec![SortField::new(element.data_type().clone())]).ok()
+ } else {
+ None
+ };
Ok(Self {
field_name: field_name.to_string(),
- java_uses_equaliser: matches!(
- element.data_type(),
- ArrowDataType::Struct(_)
- | ArrowDataType::List(_)
- | ArrowDataType::LargeList(_)
- | ArrowDataType::Map(_, _)
- | ArrowDataType::Binary
- | ArrowDataType::LargeBinary
- | ArrowDataType::FixedSizeBinary(_)
- ),
element,
distinct,
+ count_limit,
+ java_uses_equaliser,
+ key_encoder,
seen_input: false,
raw_accumulator: false,
elements: Vec::new(),
+ element_keys: None,
})
}
- fn preserve_raw(&mut self, array: &dyn Array, row_idx: usize) ->
crate::Result<()> {
+ fn limit(&self) -> usize {
+ self.count_limit.unsigned_abs() as usize
+ }
+
+ fn list<'a>(&self, array: &'a dyn Array) -> crate::Result<&'a ListArray> {
+ array
+ .as_any()
+ .downcast_ref::<ListArray>()
+ .ok_or_else(|| Error::DataInvalid {
+ message: format!(
+ "collect column '{}' requires Arrow List, got {:?}",
+ self.field_name,
+ array.data_type()
+ ),
+ source: None,
+ })
+ }
+
+ /// The input row's elements as one array, or `None` for a null cell.
+ fn input_values(&self, array: &dyn Array, row_idx: usize) ->
crate::Result<Option<ArrayRef>> {
if array.is_null(row_idx) {
- return Ok(());
+ return Ok(None);
}
- let list =
- array
- .as_any()
- .downcast_ref::<ListArray>()
- .ok_or_else(|| Error::DataInvalid {
- message: format!(
- "collect column '{}' requires Arrow List, got {:?}",
- self.field_name,
- array.data_type()
- ),
- source: None,
- })?;
- let values = list.value(row_idx);
- self.elements = (0..values.len())
- .map(|index| values.slice(index, 1))
- .collect();
- self.seen_input = true;
- self.raw_accumulator = true;
- Ok(())
+ Ok(Some(self.list(array)?.value(row_idx)))
}
- fn normalize_raw_for_agg(&mut self) {
- if !self.raw_accumulator || !self.distinct || self.java_uses_equaliser
{
- return;
+ fn concat_elements(&self, elements: &[ArrayRef]) ->
crate::Result<ArrayRef> {
+ if elements.is_empty() {
+ return Ok(new_empty_array(self.element.data_type()));
}
- let mut distinct: Vec<ArrayRef> = Vec::new();
- for element in std::mem::take(&mut self.elements) {
- if !distinct
- .iter()
- .any(|existing| existing.as_ref() == element.as_ref())
- {
- distinct.push(element);
- }
+ let slices: Vec<&dyn Array> = elements.iter().map(|e|
e.as_ref()).collect();
+ concat(&slices).map_err(|e| Error::DataInvalid {
+ message: format!(
+ "Failed to build collect result for '{}': {e}",
+ self.field_name
+ ),
+ source: Some(Box::new(e)),
+ })
+ }
+
+ fn encode_keys(&self, values: &ArrayRef) -> crate::Result<Option<Rows>> {
+ let Some(encoder) = &self.key_encoder else {
+ return Ok(None);
+ };
+ encoder
+ .convert_columns(std::slice::from_ref(values))
+ .map(Some)
+ .map_err(|e| Error::DataInvalid {
+ message: format!(
+ "Failed to encode collect elements for '{}': {e}",
+ self.field_name
+ ),
+ source: Some(Box::new(e)),
+ })
+ }
+ fn operand(&self, elements: Vec<ArrayRef>, has_duplicate: bool) ->
crate::Result<Operand> {
+ let keys = if self.key_encoder.is_some() {
+ self.encode_keys(&self.concat_elements(&elements)?)?
+ } else {
+ None
+ };
+ Ok(Operand {
+ elements,
+ keys,
+ has_duplicate,
+ })
+ }
+
+ fn input_operand(&self, values: &ArrayRef) -> crate::Result<Operand> {
+ Ok(Operand {
+ elements: (0..values.len())
+ .map(|index| values.slice(index, 1))
+ .collect(),
+ keys: self.encode_keys(values)?,
+ has_duplicate: true,
+ })
+ }
+
+ /// The accumulator as Java's merge would have collected it so far, so
+ /// that the input can be appended without rebuilding the collection.
+ fn take_collected(&mut self) -> crate::Result<Merged> {
+ let elements = std::mem::take(&mut self.elements);
+ let keys = match (self.element_keys.take(),
self.key_encoder.is_some()) {
+ (Some(keys), _) => Some(keys),
+ (None, true) => self
+ .encode_keys(&self.concat_elements(&elements)?)?
+ .map(|rows| rows.iter().map(|row|
row.as_ref().into()).collect()),
+ (None, false) => None,
+ };
+ Ok(Merged { elements, keys })
+ }
+
+ /// Java's private `agg(first, firstHasDuplicate, second,
+ /// secondHasDuplicate)`: a positive limit collects `first` then
+ /// `second`; a negative one collects both backwards, `second` first,
+ /// and reverses the result.
+ fn merge(&self, first: &Operand, second: &Operand) -> Merged {
+ let mut merged = Merged {
+ elements: Vec::new(),
+ keys: self.key_encoder.as_ref().map(|_| HashSet::new()),
+ };
+ if self.count_limit >= 0 {
+ self.collect(&mut merged, first, false);
+ self.collect(&mut merged, second, false);
+ } else {
+ self.collect(&mut merged, second, true);
+ self.collect(&mut merged, first, true);
+ merged.elements.reverse();
}
- self.elements = distinct;
- self.raw_accumulator = false;
+ merged
}
- fn collect(&mut self, array: &dyn Array, row_idx: usize) ->
crate::Result<()> {
- if array.is_null(row_idx) {
- return Ok(());
+ /// Java `collect`: add the operand's elements until the collection
+ /// holds |count-limit| of them, skipping duplicates when distinct.
+ fn collect(&self, merged: &mut Merged, operand: &Operand, backwards: bool)
{
+ // Java `checkDuplicateUntil`: with the equaliser, an operand known
+ // to be duplicate-free is checked only against what was collected
+ // before it.
+ let check_until = if operand.has_duplicate {
+ usize::MAX
+ } else {
+ merged.elements.len()
+ };
+ let len = operand.elements.len();
+ for index in (0..len).map(|i| if backwards { len - 1 - i } else { i })
{
+ if merged.elements.len() >= self.limit() {
+ break;
+ }
+ let element = &operand.elements[index];
+ if self.distinct {
+ let duplicate = match (&mut merged.keys, &operand.keys) {
+ (Some(seen), Some(rows)) =>
!seen.insert(rows.row(index).as_ref().into()),
Review Comment:
Fixed in 47799ec5 and 098baff2. FLOAT/DOUBLE set keys map every NaN to one
value before row encoding. Elements that hold FLOAT/DOUBLE values or maps at
any depth (`ROW<DOUBLE>`, `ARRAY<DOUBLE>`, `MAP`) are now compared field by
field like Java's equaliser: NaNs are one value, 0.0 and -0.0 are two, and map
entries are matched by key, so the limit can't be used up by values Java treats
as duplicates. Stored elements keep their original bits.
`distinct_doubles_merge_nan_payloads_like_java` covers your case (`[NaN(1),
NaN(2)]` then `[5.0]` with `count-limit=2` gives `[NaN(1), 5.0]`), and there
are tests for the ROW, ARRAY and MAP cases. Replays against Java master's
`FieldCollectAgg` with DOUBLE, `ROW<DOUBLE>` and `MAP<INT, DOUBLE>` elements,
including retracts, match on every step.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]