JingsongLi commented on code in PR #1387:
URL: https://github.com/apache/paimon-rust/pull/1387#discussion_r4581682361


##########
crates/paimon/src/table/aggregator/collect.rs:
##########
@@ -89,60 +109,139 @@ impl CollectAgg {
                     source: None,
                 })?;
         let values = list.value(row_idx);
-        self.elements = (0..values.len())
-            .map(|index| values.slice(index, 1))
-            .collect();
+        Ok(Some(
+            (0..values.len())
+                .map(|index| values.slice(index, 1))
+                .collect(),
+        ))
+    }
+
+    fn preserve_raw(&mut self, array: &dyn Array, row_idx: usize) -> 
crate::Result<()> {
+        let Some(elements) = self.list_elements(array, row_idx)? else {
+            return Ok(());
+        };
+        self.elements = elements;
         self.seen_input = true;
-        self.raw_accumulator = true;
         Ok(())
     }
 
-    fn normalize_raw_for_agg(&mut self) {
-        if !self.raw_accumulator || !self.distinct || self.java_uses_equaliser 
{
-            return;
+    /// Java `checkDuplicateUntil` for constructed/binary element types.
+    fn check_duplicate_until(&self, has_duplicate: bool, collection_len: 
usize) -> usize {
+        if self.distinct && self.java_uses_equaliser {
+            if has_duplicate {
+                usize::MAX
+            } else {
+                collection_len
+            }
+        } else {
+            0
         }
-        let mut distinct: Vec<ArrayRef> = Vec::new();
-        for element in std::mem::take(&mut self.elements) {
-            if !distinct
+    }
+
+    fn contains(
+        &self,
+        collection: &[ArrayRef],
+        element: &ArrayRef,
+        check_duplicate_until: usize,
+    ) -> bool {
+        if self.distinct && !self.java_uses_equaliser {
+            // LinkedHashSet path: always full content equality.
+            return collection
                 .iter()
-                .any(|existing| existing.as_ref() == element.as_ref())
-            {
-                distinct.push(element);
+                .any(|existing| existing.as_ref() == element.as_ref());
+        }
+        if check_duplicate_until == 0 {
+            return false;
+        }
+        collection
+            .iter()
+            .take(check_duplicate_until)
+            .any(|existing| existing.as_ref() == element.as_ref())
+    }
+
+    fn collect_into(
+        &self,
+        collection: &mut Vec<ArrayRef>,
+        elements: &[ArrayRef],
+        from_end: bool,
+        check_duplicate_until: usize,
+    ) {
+        let limit = self.abs_limit();
+        let mut i = 0usize;
+        while i < elements.len() && (collection.len() as i64) < limit {
+            let index = if from_end { elements.len() - 1 - i } else { i };
+            let element = elements[index].clone();
+            if !self.contains(collection, &element, check_duplicate_until) {
+                collection.push(element);
             }
+            i += 1;
         }
-        self.elements = distinct;
-        self.raw_accumulator = false;
     }
 
-    fn collect(&mut self, array: &dyn Array, row_idx: usize) -> 
crate::Result<()> {
-        if array.is_null(row_idx) {
-            return Ok(());
+    /// Java `FieldCollectAgg#agg(accumulator, accumulatorHasDuplicate, input, 
inputHasDuplicate)`.
+    fn merge(
+        &mut self,
+        accumulator: Option<Vec<ArrayRef>>,
+        accumulator_has_duplicate: bool,
+        input: Option<Vec<ArrayRef>>,
+        input_has_duplicate: bool,
+    ) {
+        if accumulator.is_none() && input.is_none() {
+            return;
         }
-        let list =
-            array
-                .as_any()
-                .downcast_ref::<ListArray>()
-                .ok_or_else(|| Error::DataInvalid {
-                    message: format!(
-                        "collect column '{}' requires Arrow List, got {:?}",
-                        self.field_name,
-                        array.data_type()
-                    ),
-                    source: None,
-                })?;
-        self.seen_input = true;
-        let values = list.value(row_idx);
-        for index in 0..values.len() {
-            let element = values.slice(index, 1);
-            if !self.distinct
-                || !self
-                    .elements
-                    .iter()
-                    .any(|existing| existing.as_ref() == element.as_ref())
-            {
-                self.elements.push(element);
+
+        let abs_limit = self.abs_limit();
+        if let (None, Some(input_elements)) = (&accumulator, &input) {
+            if !self.distinct && (input_elements.len() as i64) <= abs_limit {
+                self.elements = input_elements.clone();
+                self.seen_input = true;
+                return;
+            }
+        }
+        if let (Some(acc_elements), None) = (&accumulator, &input) {
+            if !self.distinct && (acc_elements.len() as i64) <= abs_limit {
+                self.elements = acc_elements.clone();
+                self.seen_input = true;
+                return;
             }
         }
+
+        let empty = Vec::new();
+        let acc_ref = accumulator.as_ref().unwrap_or(&empty);
+        let input_ref = input.as_ref().unwrap_or(&empty);
+        let mut collection = Vec::new();
+        if self.count_limit >= 0 {
+            let until = self.check_duplicate_until(accumulator_has_duplicate, 
collection.len());
+            self.collect_into(&mut collection, acc_ref, false, until);

Review Comment:
   [P1] Keep the default collect accumulator incremental
   
   With distinct=false and no count-limit configured, every singleton input now 
takes the existing accumulator, allocates a new collection and copies all 
previous elements through collect_into. N same-key rows therefore perform O(N²) 
work, whereas the base appends in place. I reproduced this through actual table 
write → prepare_commit → commit → read: 20,000 singleton ARRAY<INT> rows take 
~2002 ms on this head versus ~21 ms on the exact f91fdb8a base, with the same 
correct result (debug builds; this is a comparative regression probe, not a 
production throughput claim). At 2,000 rows the head is ~36 ms. This affects 
existing tables even when they do not enable the new option. Retain/reuse the 
accumulator for append/prepend and bound it as needed rather than 
reconstructing it on every merge.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to