ZZZxDong commented on code in PR #10983:
URL: https://github.com/apache/paimon/pull/10983#discussion_r4464418499


##########
paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java:
##########
@@ -1361,17 +1363,72 @@ private static void 
validateMergeFunctionFactory(TableSchema schema) {
             default:
         }
 
-        for (int i = 0; i < schema.logicalRowType().getFieldNames().size(); 
i++) {
-            String fieldName = schema.logicalRowType().getFieldNames().get(i);
-            String aggFuncName = options.fieldAggFunc(fieldName);
-            aggFuncName = aggFuncName == null ? options.fieldsDefaultFunc() : 
aggFuncName;
-            if (aggFuncName != null) {
-                FactoryUtil.discoverFactory(
-                        FieldAggregator.class.getClassLoader(),
-                        FieldAggregatorFactory.class,
-                        aggFuncName);
+        RowType rowType = schema.logicalRowType();
+        List<String> primaryKeys = schema.primaryKeys();
+        List<String> sequenceFields = options.sequenceField();
+        Set<String> excludedFields = new HashSet<>();
+        if (options.mergeEngine() == MergeEngine.PARTIAL_UPDATE) {
+            // Sequence-group ordering fields never get an aggregator.
+            excludedFields.addAll(allSequenceGroupOrderingFields(options));
+        }
+        for (int i = 0; i < rowType.getFieldNames().size(); i++) {
+            String fieldName = rowType.getFieldNames().get(i);
+            if (excludedFields.contains(fieldName)) {
+                continue;
+            }
+            String aggFuncName =
+                    resolveAggFuncName(options, fieldName, primaryKeys, 
sequenceFields);
+            if (aggFuncName == null) {
+                continue;
+            }
+            // Instantiate once so that type-incompatible configurations (e.g. 
'collect' on a
+            // non-array column) are rejected at DDL time instead of at 
read/write time.
+            FieldAggregatorFactory.create(
+                    rowType.getFieldTypes().get(i), fieldName, aggFuncName, 
options);
+        }
+    }
+
+    /**
+     * The fields named in the option keys of all sequence groups, e.g. {@code 
f2} for {@code
+     * fields.f2.sequence-group = f3}. They order the group instead of being 
aggregated.
+     */
+    private static List<String> allSequenceGroupOrderingFields(CoreOptions 
options) {
+        List<String> orderingFields = new ArrayList<>();
+        for (String optionKey : options.toMap().keySet()) {
+            if (isSequenceGroupOption(optionKey)) {
+                orderingFields.addAll(sequenceGroupOrderingFields(optionKey));
             }
         }
+        return orderingFields;
+    }
+
+    /**
+     * Mirrors how each merge engine resolves the aggregation function of a 
field, so that the
+     * aggregation factory can be instantiated once at DDL time.
+     */
+    @Nullable
+    private static String resolveAggFuncName(
+            CoreOptions options,
+            String fieldName,
+            List<String> primaryKeys,
+            List<String> sequenceFields) {
+        if (options.mergeEngine() == MergeEngine.AGGREGATE) {
+            return AggregateMergeFunction.getAggFuncName(
+                    fieldName, options, primaryKeys, sequenceFields);
+        }
+
+        if (sequenceFields.contains(fieldName)) {

Review Comment:
   You're right — sequence.field columns aren't sequence-group ordering fields, 
so they still inherit the default aggregate function; I now reuse 
PartialUpdateMergeFunction#getAggFuncName so DDL validation matches runtime, 
and added a regression test for it.



##########
paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java:
##########
@@ -1361,17 +1363,72 @@ private static void 
validateMergeFunctionFactory(TableSchema schema) {
             default:
         }
 
-        for (int i = 0; i < schema.logicalRowType().getFieldNames().size(); 
i++) {
-            String fieldName = schema.logicalRowType().getFieldNames().get(i);
-            String aggFuncName = options.fieldAggFunc(fieldName);
-            aggFuncName = aggFuncName == null ? options.fieldsDefaultFunc() : 
aggFuncName;
-            if (aggFuncName != null) {
-                FactoryUtil.discoverFactory(
-                        FieldAggregator.class.getClassLoader(),
-                        FieldAggregatorFactory.class,
-                        aggFuncName);
+        RowType rowType = schema.logicalRowType();
+        List<String> primaryKeys = schema.primaryKeys();
+        List<String> sequenceFields = options.sequenceField();
+        Set<String> excludedFields = new HashSet<>();
+        if (options.mergeEngine() == MergeEngine.PARTIAL_UPDATE) {
+            // Sequence-group ordering fields never get an aggregator.
+            excludedFields.addAll(allSequenceGroupOrderingFields(options));
+        }
+        for (int i = 0; i < rowType.getFieldNames().size(); i++) {
+            String fieldName = rowType.getFieldNames().get(i);
+            if (excludedFields.contains(fieldName)) {
+                continue;
+            }
+            String aggFuncName =
+                    resolveAggFuncName(options, fieldName, primaryKeys, 
sequenceFields);
+            if (aggFuncName == null) {
+                continue;
+            }
+            // Instantiate once so that type-incompatible configurations (e.g. 
'collect' on a
+            // non-array column) are rejected at DDL time instead of at 
read/write time.
+            FieldAggregatorFactory.create(
+                    rowType.getFieldTypes().get(i), fieldName, aggFuncName, 
options);
+        }
+    }
+
+    /**
+     * The fields named in the option keys of all sequence groups, e.g. {@code 
f2} for {@code
+     * fields.f2.sequence-group = f3}. They order the group instead of being 
aggregated.
+     */
+    private static List<String> allSequenceGroupOrderingFields(CoreOptions 
options) {
+        List<String> orderingFields = new ArrayList<>();
+        for (String optionKey : options.toMap().keySet()) {
+            if (isSequenceGroupOption(optionKey)) {
+                orderingFields.addAll(sequenceGroupOrderingFields(optionKey));
             }
         }
+        return orderingFields;
+    }
+
+    /**
+     * Mirrors how each merge engine resolves the aggregation function of a 
field, so that the
+     * aggregation factory can be instantiated once at DDL time.
+     */
+    @Nullable
+    private static String resolveAggFuncName(
+            CoreOptions options,
+            String fieldName,
+            List<String> primaryKeys,
+            List<String> sequenceFields) {
+        if (options.mergeEngine() == MergeEngine.AGGREGATE) {
+            return AggregateMergeFunction.getAggFuncName(
+                    fieldName, options, primaryKeys, sequenceFields);
+        }
+
+        if (sequenceFields.contains(fieldName)) {

Review Comment:
   You're right — sequence.field columns aren't sequence-group ordering fields, 
so they still inherit the default aggregate function; I now reuse 
PartialUpdateMergeFunction#getAggFuncName so DDL validation matches runtime, 
and added a regression test for it.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to