This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 622e67d8c2 [core] Ignore blank values in FieldListaggAgg operator.
(#8480)
622e67d8c2 is described below
commit 622e67d8c28c5c529ea09b8575cb2e0e9d849222
Author: AuroraVoyage <[email protected]>
AuthorDate: Tue Jul 7 15:21:37 2026 +0800
[core] Ignore blank values in FieldListaggAgg operator. (#8480)
---
.../compact/aggregate/FieldListaggAgg.java | 19 ++---
.../compact/aggregate/FieldAggregatorTest.java | 86 ++++++++++++++++++++++
2 files changed, 94 insertions(+), 11 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java
index 6ca31bbead..364993b558 100644
---
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java
+++
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java
@@ -22,6 +22,7 @@ import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.BinaryStringUtils;
+import org.apache.paimon.utils.StringUtils;
import java.util.ArrayList;
import java.util.HashSet;
@@ -50,21 +51,17 @@ public class FieldListaggAgg extends FieldAggregator {
@Override
public Object agg(Object accumulator, Object inputField) {
- if (accumulator == null || inputField == null) {
- return accumulator == null ? inputField : accumulator;
+ if (inputField == null || StringUtils.isBlank(inputField.toString())) {
+ return accumulator;
}
// ordered by type root definition
- BinaryString mergeFieldSD = (BinaryString) accumulator;
- BinaryString inFieldSD = (BinaryString) inputField;
-
- if (inFieldSD.getSizeInBytes() <= 0) {
- return mergeFieldSD;
+ if (accumulator == null ||
StringUtils.isBlank(accumulator.toString())) {
+ return inputField;
}
- if (mergeFieldSD.getSizeInBytes() <= 0) {
- return inFieldSD;
- }
+ BinaryString mergeFieldSD = (BinaryString) accumulator;
+ BinaryString inFieldSD = (BinaryString) inputField;
if (distinct) {
BinaryString[] accumulatorTokens =
@@ -78,7 +75,7 @@ public class FieldListaggAgg extends FieldAggregator {
result.add(mergeFieldSD);
for (BinaryString str :
splitByWholeSeparatorPreserveAllTokens(inFieldSD,
delimiterBinaryString)) {
- if (str.getSizeInBytes() == 0 || existingTokens.contains(str))
{
+ if (StringUtils.isBlank(str.toString()) ||
existingTokens.contains(str)) {
continue;
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java
index 183a1b63ef..5f9537e5a7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java
@@ -411,6 +411,92 @@ public class FieldAggregatorTest {
"Data type for list agg column must be STRING
(unbounded VARCHAR), but was VARCHAR(100)");
}
+ @Test
+ public void testFieldListAggWithDefaultDelimiterIgnoringBlankValues() {
+ FieldListaggAgg fieldListaggAgg =
+ new FieldListaggAggFactory()
+ .create(
+ new VarCharType(VarCharType.MAX_LENGTH),
+ new CoreOptions(new HashMap<>()),
+ "fieldName");
+ BinaryString result =
+ Stream.of(
+ BinaryString.fromString("user1"),
+ BinaryString.fromString(""),
+ BinaryString.fromString(" "),
+ BinaryString.fromString(" "),
+ BinaryString.fromString("\t"),
+ BinaryString.fromString("\n"),
+ BinaryString.fromString("\r"),
+ BinaryString.fromString("\r\n"),
+ BinaryString.fromString(" \t\n "),
+ BinaryString.fromString(" \t\n\r\n \u3000 "),
+ BinaryString.fromString("user2"),
+ BinaryString.fromString("\u3000"),
+ BinaryString.fromString("\u2000"))
+ .sequential()
+ .reduce((l, r) -> (BinaryString)
fieldListaggAgg.agg(l, r))
+ .orElse(null);
+
+ assertNotNull(result);
+ assertThat(result.toString()).isEqualTo("user1,user2");
+ }
+
+ @Test
+ public void
testFieldListAggWithDefaultDelimiterAndDistinctIgnoringBlankValues() {
+ FieldListaggAgg fieldListaggAgg =
+ new FieldListaggAggFactory()
+ .create(
+ new VarCharType(VarCharType.MAX_LENGTH),
+ CoreOptions.fromMap(
+
ImmutableMap.of("fields.fieldName.distinct", "true")),
+ "fieldName");
+
+ BinaryString result =
+ Stream.of(
+ BinaryString.fromString("user1"),
+ BinaryString.fromString("user2"),
+ BinaryString.fromString("user1"),
+ BinaryString.fromString("user3"),
+ BinaryString.fromString(""),
+ BinaryString.fromString(" "),
+ BinaryString.fromString(" "),
+ BinaryString.fromString("\t"),
+ BinaryString.fromString("\n"),
+ BinaryString.fromString("\r"),
+ BinaryString.fromString("\r\n"),
+ BinaryString.fromString(" \t\n "),
+ BinaryString.fromString(" \t\n\r\n \u3000 "),
+ BinaryString.fromString("user2"),
+ BinaryString.fromString("user3"),
+ BinaryString.fromString("\u3000"),
+ BinaryString.fromString("\u2000"))
+ .sequential()
+ .reduce((l, r) -> (BinaryString)
fieldListaggAgg.agg(l, r))
+ .orElse(null);
+
+ assertNotNull(result);
+ assertEquals("user1,user2,user3", result.toString());
+ }
+
+ @Test
+ public void testFieldListAggFirstNonBlankValueWithoutLeadingDelimiter() {
+ FieldListaggAgg fieldListaggAgg =
+ new FieldListaggAggFactory()
+ .create(
+ new VarCharType(VarCharType.MAX_LENGTH),
+ new CoreOptions(new HashMap<>()),
+ "fieldName");
+
+ BinaryString accumulator = null;
+ accumulator = (BinaryString) fieldListaggAgg.agg(accumulator,
BinaryString.fromString(" "));
+ accumulator =
+ (BinaryString)
+ fieldListaggAgg.agg(accumulator,
BinaryString.fromString("first line"));
+
+ assertThat(accumulator.toString()).isEqualTo("first line");
+ }
+
@Test
public void testFieldMaxAgg() {
FieldMaxAgg fieldMaxAgg = new FieldMaxAggFactory().create(new
IntType(), null, null);
