This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch routineload_flexible_update
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/routineload_flexible_update by
this push:
new a64cdd99f80 fix: Handle conflicting partial_columns and
unique_key_update_mode settings
a64cdd99f80 is described below
commit a64cdd99f808a4aabb5df378ee9adc49c3227afa
Author: Yongqiang YANG <[email protected]>
AuthorDate: Thu Jan 15 21:58:14 2026 -0800
fix: Handle conflicting partial_columns and unique_key_update_mode settings
Address review comments:
- Reset uniqueKeyUpdateMode to UPSERT when partial_columns is set to false
but mode was UPDATE_FIXED_COLUMNS (in modifyCommonJobProperties)
- Add backward compatibility handling in gsonPostProcess for inconsistent
metadata states
- Add validation to reject partial_columns=false with UPDATE_FIXED_COLUMNS
or UPDATE_FLEXIBLE_COLUMNS modes in CreateRoutineLoadInfo
---
.../doris/load/routineload/RoutineLoadJob.java | 12 ++++++++++-
.../plans/commands/info/CreateRoutineLoadInfo.java | 25 ++++++++++++++++------
2 files changed, 29 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index 585c3167154..b098fbba662 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -1957,12 +1957,19 @@ public abstract class RoutineLoadJob
// Process remaining properties
jobProperties.forEach((k, v) -> {
if (k.equals(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) {
+ boolean partialColumns = Boolean.parseBoolean(v);
// Backward compatibility: if unique_key_update_mode is not
set, use partial_columns
if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) {
- isPartialUpdate = Boolean.parseBoolean(v);
+ isPartialUpdate = partialColumns;
if (isPartialUpdate) {
uniqueKeyUpdateMode =
TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS;
}
+ } else if (!partialColumns
+ && uniqueKeyUpdateMode ==
TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) {
+ // Backward compatibility: if partial_columns is false but
unique_key_update_mode
+ // was UPDATE_FIXED_COLUMNS, reset to UPSERT for
consistency
+ isPartialUpdate = false;
+ uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT;
}
} else if
(k.equals(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) {
if ("ERROR".equalsIgnoreCase(v)) {
@@ -2058,6 +2065,9 @@ public abstract class RoutineLoadJob
jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_COLUMNS));
if (this.isPartialUpdate && uniqueKeyUpdateMode ==
TUniqueKeyUpdateMode.UPSERT) {
this.uniqueKeyUpdateMode =
TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS;
+ } else if (!this.isPartialUpdate && uniqueKeyUpdateMode ==
TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) {
+ // Reset to UPSERT when partial_columns is disabled
+ this.uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT;
}
this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS,
String.valueOf(isPartialUpdate));
this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE,
uniqueKeyUpdateMode.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
index 9394257013c..ae2781c5abd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
@@ -630,13 +630,24 @@ public class CreateRoutineLoadInfo {
if (jobProperties.containsKey(UNIQUE_KEY_UPDATE_MODE)) {
String modeStr = jobProperties.get(UNIQUE_KEY_UPDATE_MODE);
TUniqueKeyUpdateMode mode =
parseAndValidateUniqueKeyUpdateMode(modeStr);
- // Check for conflicting settings: partial_columns=true with
unique_key_update_mode=UPSERT
- if (jobProperties.containsKey(PARTIAL_COLUMNS)
- &&
jobProperties.get(PARTIAL_COLUMNS).equalsIgnoreCase("true")
- && mode == TUniqueKeyUpdateMode.UPSERT) {
- throw new AnalysisException("Cannot set both
'partial_columns=true' and "
- + "'unique_key_update_mode=UPSERT'. "
- + "Use unique_key_update_mode=UPDATE_FIXED_COLUMNS
instead.");
+ // Check for conflicting settings between partial_columns and
unique_key_update_mode
+ if (jobProperties.containsKey(PARTIAL_COLUMNS)) {
+ String partialColumnsVal = jobProperties.get(PARTIAL_COLUMNS);
+ // partial_columns=true is not allowed with UPSERT
+ if (partialColumnsVal.equalsIgnoreCase("true")
+ && mode == TUniqueKeyUpdateMode.UPSERT) {
+ throw new AnalysisException("Cannot set both
'partial_columns=true' and "
+ + "'unique_key_update_mode=UPSERT'. "
+ + "Use unique_key_update_mode=UPDATE_FIXED_COLUMNS
instead.");
+ }
+ // partial_columns=false is not allowed with
UPDATE_FIXED_COLUMNS or UPDATE_FLEXIBLE_COLUMNS
+ if (partialColumnsVal.equalsIgnoreCase("false")
+ && (mode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS
+ || mode ==
TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS)) {
+ throw new AnalysisException("Cannot set
'partial_columns=false' when "
+ + "'unique_key_update_mode' is
'UPDATE_FIXED_COLUMNS' or 'UPDATE_FLEXIBLE_COLUMNS'. "
+ + "Use unique_key_update_mode=UPSERT instead, or
enable partial columns.");
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
