github-actions[bot] commented on code in PR #66685:
URL: https://github.com/apache/doris/pull/66685#discussion_r3791982938
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java:
##########
@@ -168,6 +192,106 @@ static boolean requiresSingleWriter(FileStoreTable
paimonTable) {
==
CoreOptions.ChangelogProducer.FULL_COMPACTION);
}
+ private DistributionSpecPaimonTableSinkHashPartitioned
buildFixedBucketDistributionSpec(
+ FileStoreTable paimonTable) {
+ if (Config.be_exec_version
+ <
DistributionSpecExternalTableSinkHashPartitioned.MIN_BE_EXEC_VERSION) {
+ return null;
+ }
+ return buildFixedBucketDistributionSpec(paimonTable, cols,
child().getOutput());
+ }
+
+ static DistributionSpecPaimonTableSinkHashPartitioned
buildFixedBucketDistributionSpec(
+ FileStoreTable paimonTable, List<Column> sinkColumns, List<Slot>
sinkOutput) {
+ if (paimonTable.bucketMode() != BucketMode.HASH_FIXED) {
+ return null;
+ }
+
+ TableSchema schema = paimonTable.schema();
+ CoreOptions coreOptions = CoreOptions.fromMap(schema.options());
+ if (coreOptions.bucketFunctionType() !=
CoreOptions.BucketFunctionType.DEFAULT) {
+ return null;
+ }
+
+ if (schema.numBuckets() <= 0 || schema.bucketKeys().isEmpty()
+ || sinkColumns.size() != sinkOutput.size()) {
+ return null;
+ }
+
+ Map<String, Slot> outputByName = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+ for (int i = 0; i < sinkColumns.size(); i++) {
+ if (outputByName.put(sinkColumns.get(i).getName(),
sinkOutput.get(i)) != null) {
+ return null;
+ }
+ }
+ Map<String, DataField> fieldsByName = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+ for (DataField field : schema.fields()) {
+ fieldsByName.put(field.name(), field);
+ }
+
+ List<ExprId> routeExprIds = new ArrayList<>();
+ Map<String, Integer> routeIndexes = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+ List<Integer> partitionFieldIndexes = appendRouteFields(
+ schema.partitionKeys(), outputByName, fieldsByName,
routeExprIds, routeIndexes);
+ List<Integer> bucketFieldIndexes = appendRouteFields(
+ schema.bucketKeys(), outputByName, fieldsByName, routeExprIds,
routeIndexes);
+ if (partitionFieldIndexes == null || bucketFieldIndexes == null
+ || bucketFieldIndexes.isEmpty()) {
+ return null;
+ }
+ return new DistributionSpecPaimonTableSinkHashPartitioned(
+ routeExprIds, schema.numBuckets(), partitionFieldIndexes,
bucketFieldIndexes);
+ }
+
+ private static List<Integer> appendRouteFields(List<String> fieldNames,
+ Map<String, Slot> outputByName, Map<String, DataField>
fieldsByName,
+ List<ExprId> routeExprIds, Map<String, Integer> routeIndexes) {
Review Comment:
[P1] Fall back when a Paimon route field has a default
This admits nullable partition/bucket fields based only on their type, but
the two sides do not hash the same logical value for an explicit `NULL`. The BE
exchange hashes the null as-is; [Paimon 1.4.2's
`TableWriteImpl`](https://github.com/apache/paimon/blob/release-1.4.2/paimon-core/src/main/java/org/apache/paimon/table/sink/TableWriteImpl.java#L183-L190)
then applies `DefaultValueRow` before extracting the partition and bucket,
replacing that null with the schema default. For example, with a nullable `INT
DEFAULT 1` bucket key and four writers, the checked-in hashes route NULL
(bucket 0) and literal 1 (bucket 2) to different writers even though both are
ultimately written to bucket 2. In the normal `write-only=false` append mode,
both sessions may then restore and compact the same prior bucket files. Please
reject route fields with non-null defaults (using the existing safe fallback)
until the exchange applies Paimon's converted defaults, and cover explicit NULL
for defaulted bu
cket and partition keys.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
