This is an automated email from the ASF dual-hosted git repository.
bobhan1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c02808aae79 [fix](fe) Handle generated columns in delete partial
update (#64884)
c02808aae79 is described below
commit c02808aae79b29e8c02e48f06baf89b43d4c4152
Author: bobhan1 <[email protected]>
AuthorDate: Tue Jun 30 16:37:50 2026 +0800
[fix](fe) Handle generated columns in delete partial update (#64884)
### What problem does this PR solve?
Issue Number: N/A
Related PR: N/A
Original Problem: DELETE failed during analysis on a unique
merge-on-write table that contains a generated column when light delete
is disabled. The original reported table had a generated VARIANT column:
```sql
new_col variant AS (receive_address_detail) NULL
```
and the DELETE failed with:
```text
errCode = 2, detailMessage = cannot find column from target table
/receive_address_detail
```
A minimized reproduction is:
```sql
create table test_gen_col_mow_delete_partial_update (
a int,
b int,
c int as (b + 1),
d int
)
unique key(a)
distributed by hash(a) buckets 1
properties (
"enable_unique_key_merge_on_write" = "true",
"enable_mow_light_delete" = "false",
"replication_num" = "1"
);
```
After inserting rows, running:
```sql
delete from test_gen_col_mow_delete_partial_update where a = 1;
```
failed with:
```text
java.sql.SQLException: errCode = 2, detailMessage = cannot find column from
target table [b]
```
Problem Summary: DELETE on a unique merge-on-write table with light
delete disabled is rewritten to a partial update load that writes key
columns and the delete sign. BindSink used to auto-add every generated
column and then recompute omitted generated columns, which required
ordinary value columns that were not part of the DELETE output and
caused analysis to fail.
This change treats DELETE partial updates specially: generated columns
that are not emitted by the child plan are skipped, and generated
columns that are emitted by the child plan use that child output
directly. Normal partial update generated-column dependency checks are
unchanged.
The tests cover generated value columns, generated key columns, a
generated value-column table with an omitted NOT NULL value column that
has no default value, and the original VARIANT generated-column shape
based on `receive_address_detail`.
### Release note
Fix DELETE failure on unique merge-on-write tables with generated
columns when light delete is disabled.
### Check List (For Author)
- Test: Regression test / Unit Test
- `./build.sh --fe -j100`
- `./run-fe-ut.sh --run
org.apache.doris.nereids.trees.plans.DeleteFromUsingCommandTest`
- `./run-regression-test.sh --run -d
regression-test/suites/ddl_p0/test_create_table_generated_column -s
test_generated_column_delete -forceGenOut`
- `./run-regression-test.sh --run -d
regression-test/suites/ddl_p0/test_create_table_generated_column -s
test_generated_column_delete`
- `./run-regression-test.sh --run -d
regression-test/suites/ddl_p0/test_create_table_generated_column -s
test_partial_update_generated_column`
- Behavior changed: Yes (DELETE now succeeds for unique merge-on-write
tables with generated columns when light delete is disabled.)
- Does this need documentation: No
---
.../doris/nereids/rules/analysis/BindSink.java | 30 +++++--
.../trees/plans/DeleteFromUsingCommandTest.java | 89 ++++++++++++++++++++
.../test_delete_generated_column.out | 12 +++
.../test_delete_generated_column.groovy | 96 ++++++++++++++++++++++
4 files changed, 219 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
index d7da7498ba7..8c580171b21 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
@@ -183,6 +183,7 @@ public class BindSink implements AnalysisRuleFactory {
DatabaseIf database = pair.first;
OlapTable table = pair.second;
boolean isPartialUpdate = sink.isPartialUpdate() &&
table.getKeysType() == KeysType.UNIQUE_KEYS;
+ boolean isDeletePartialUpdate = isPartialUpdate &&
sink.getDMLCommandType() == DMLCommandType.DELETE;
TPartialUpdateNewRowPolicy partialUpdateNewKeyPolicy =
sink.getPartialUpdateNewRowPolicy();
LogicalPlan child = ((LogicalPlan) sink.child());
@@ -194,7 +195,7 @@ public class BindSink implements AnalysisRuleFactory {
// 1. bind target columns: from sink's column names to target tables'
Columns
Pair<List<Column>, Integer> bindColumnsResult =
bindTargetColumns(table, sink.getColNames(), childHasSeqCol,
needExtraSeqCol,
- sink.getDMLCommandType() ==
DMLCommandType.GROUP_COMMIT);
+ sink.getDMLCommandType() ==
DMLCommandType.GROUP_COMMIT, isDeletePartialUpdate);
List<Column> bindColumns = bindColumnsResult.first;
int extraColumnsNum = bindColumnsResult.second;
@@ -279,7 +280,7 @@ public class BindSink implements AnalysisRuleFactory {
}
Map<String, NamedExpression> columnToOutput = getColumnToOutput(
- ctx, table, isPartialUpdate, boundSink, child);
+ ctx, table, isPartialUpdate, isDeletePartialUpdate, boundSink,
child);
LogicalProject<?> fullOutputProject = getOutputProjectByCoercion(
table.getFullSchema(), child, columnToOutput);
List<Column> columns = new ArrayList<>(table.getFullSchema().size());
@@ -371,7 +372,8 @@ public class BindSink implements AnalysisRuleFactory {
private static Map<String, NamedExpression> getColumnToOutput(
MatchingContext<? extends UnboundLogicalSink<Plan>> ctx,
- TableIf table, boolean isPartialUpdate, LogicalTableSink<?>
boundSink, LogicalPlan child) {
+ TableIf table, boolean isPartialUpdate, boolean
isDeletePartialUpdate,
+ LogicalTableSink<?> boundSink, LogicalPlan child) {
// we need to insert all the columns of the target table
// although some columns are not mentions.
// so we add a projects to supply the default value.
@@ -494,6 +496,18 @@ public class BindSink implements AnalysisRuleFactory {
// if processed in upper for loop, will lead to not found slot error
// It's the same reason for moving the processing of materialized
columns down.
for (Column column : generatedColumns) {
+ if (isDeletePartialUpdate) {
+ NamedExpression childOutput = columnToChildOutput.get(column);
+ if (childOutput == null) {
+ continue;
+ }
+ Alias output = new Alias(TypeCoercionUtils.castIfNotSameType(
+ childOutput,
DataType.fromCatalogType(column.getType())), column.getName());
+ columnToOutput.put(column.getName(), output);
+ columnToReplaced.put(column.getName(), output.toSlot());
+ replaceMap.put(output.toSlot(), output.child());
+ continue;
+ }
Map<String, String> currentSessionVars =
ctx.connectContext.getSessionVariable().getAffectQueryResultInPlanVariables();
try (AutoCloseSessionVariable autoClose = new
AutoCloseSessionVariable(ctx.connectContext,
@@ -704,7 +718,7 @@ public class BindSink implements AnalysisRuleFactory {
if (boundSink.getCols().size() != child.getOutput().size()) {
throw new AnalysisException("insert into cols should be
corresponding to the query output");
}
- Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx,
table, false,
+ Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx,
table, false, false,
boundSink, child);
LogicalProject<?> fullOutputProject =
getOutputProjectByCoercion(table.getFullSchema(), child, columnToOutput);
return boundSink.withChildAndUpdateOutput(fullOutputProject);
@@ -780,7 +794,7 @@ public class BindSink implements AnalysisRuleFactory {
+ "Expected " + boundSink.getCols().size() + " columns but
got " + child.getOutput().size());
}
- Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx,
table, false,
+ Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx,
table, false, false,
boundSink, child);
// For static partition columns, add constant expressions from
PARTITION clause
@@ -905,7 +919,7 @@ public class BindSink implements AnalysisRuleFactory {
if (boundSink.getCols().size() != child.getOutput().size()) {
throw new AnalysisException("insert into cols should be
corresponding to the query output");
}
- Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx,
table, false,
+ Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx,
table, false, false,
boundSink, child);
LogicalProject<?> fullOutputProject =
getOutputProjectByCoercion(table.getFullSchema(), child, columnToOutput);
return boundSink.withChildAndUpdateOutput(fullOutputProject);
@@ -1132,7 +1146,7 @@ public class BindSink implements AnalysisRuleFactory {
// bindTargetColumns means bind sink node's target columns' names to
target table's columns
private Pair<List<Column>, Integer> bindTargetColumns(OlapTable table,
List<String> colsName,
- boolean childHasSeqCol, boolean needExtraSeqCol, boolean
isGroupCommit) {
+ boolean childHasSeqCol, boolean needExtraSeqCol, boolean
isGroupCommit, boolean isDeletePartialUpdate) {
// if the table set sequence column in stream load phase, the sequence
map column is null, we query it.
if (colsName.isEmpty()) {
// ATTN: group commit without column list should return all base
index column
@@ -1150,7 +1164,7 @@ public class BindSink implements AnalysisRuleFactory {
++extraColumnsNum;
processedColsName.add(col.getName());
}
- } else if (col.isGeneratedColumn()) {
+ } else if (col.isGeneratedColumn() && !isDeletePartialUpdate) {
++extraColumnsNum;
processedColsName.add(col.getName());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteFromUsingCommandTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteFromUsingCommandTest.java
index bf5f660e837..04531e50999 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteFromUsingCommandTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteFromUsingCommandTest.java
@@ -68,6 +68,60 @@ public class DeleteFromUsingCommandTest extends
TestWithFeService implements Pla
+ "properties(\n"
+ " \"replication_num\"=\"1\"\n"
+ ")");
+ createTable("create table gen_value (\n"
+ + " a int,\n"
+ + " b int,\n"
+ + " c int as (b + 1),\n"
+ + " d int\n"
+ + ")\n"
+ + "unique key(a)\n"
+ + "distributed by hash(a) buckets 4\n"
+ + "properties(\n"
+ + " \"replication_num\"=\"1\",\n"
+ + " \"enable_unique_key_merge_on_write\" = \"true\",\n"
+ + " \"enable_mow_light_delete\" = \"false\"\n"
+ + ")");
+ createTable("create table gen_key (\n"
+ + " a int,\n"
+ + " c int as (b + 1),\n"
+ + " b int,\n"
+ + " d int\n"
+ + ")\n"
+ + "unique key(a, c)\n"
+ + "distributed by hash(a) buckets 4\n"
+ + "properties(\n"
+ + " \"replication_num\"=\"1\",\n"
+ + " \"enable_unique_key_merge_on_write\" = \"true\",\n"
+ + " \"enable_mow_light_delete\" = \"false\"\n"
+ + ")");
+ createTable("create table gen_value_required (\n"
+ + " a int,\n"
+ + " b int,\n"
+ + " c int as (b + 1),\n"
+ + " d int not null\n"
+ + ")\n"
+ + "unique key(a)\n"
+ + "distributed by hash(a) buckets 4\n"
+ + "properties(\n"
+ + " \"replication_num\"=\"1\",\n"
+ + " \"enable_unique_key_merge_on_write\" = \"true\",\n"
+ + " \"enable_mow_light_delete\" = \"false\"\n"
+ + ")");
+ createTable("create table gen_variant (\n"
+ + " id int not null,\n"
+ + " create_time datetime not null,\n"
+ + " order_no varchar(128) not null,\n"
+ + " receive_address_detail varchar(1024) not null default
\"{}\",\n"
+ + " d int not null,\n"
+ + " new_col variant as (receive_address_detail) null\n"
+ + ")\n"
+ + "unique key(id, create_time, order_no)\n"
+ + "distributed by hash(order_no) buckets 4\n"
+ + "properties(\n"
+ + " \"replication_num\"=\"1\",\n"
+ + " \"enable_unique_key_merge_on_write\" = \"true\",\n"
+ + " \"enable_mow_light_delete\" = \"false\"\n"
+ + ")");
}
@Test
@@ -106,4 +160,39 @@ public class DeleteFromUsingCommandTest extends
TestWithFeService implements Pla
)
);
}
+
+ @Test
+ public void testDeletePartialUpdateWithGeneratedValueColumn() {
+ assertDeletePartialUpdateAnalyze("delete from gen_value t using src
where t.a = src.k1");
+ }
+
+ @Test
+ public void testDeletePartialUpdateWithGeneratedKeyColumn() {
+ assertDeletePartialUpdateAnalyze("delete from gen_key t using src
where t.a = src.k1");
+ }
+
+ @Test
+ public void testDeletePartialUpdateWithNotNullValueColumn() {
+ assertDeletePartialUpdateAnalyze("delete from gen_value_required t
using src where t.a = src.k1");
+ }
+
+ @Test
+ public void testDeletePartialUpdateWithVariantGeneratedColumn() {
+ assertDeletePartialUpdateAnalyze("delete from gen_variant t using src
where t.id = src.k1");
+ }
+
+ private void assertDeletePartialUpdateAnalyze(String sql) {
+ LogicalPlan parsed = new NereidsParser().parseSingle(sql);
+ Assertions.assertInstanceOf(DeleteFromUsingCommand.class, parsed);
+ DeleteFromUsingCommand command = ((DeleteFromUsingCommand) parsed);
+ LogicalPlan plan = command.completeQueryPlan(connectContext,
command.getLogicalQuery());
+ PlanChecker.from(connectContext, plan)
+ .analyze(plan)
+ .rewrite()
+ .matches(
+ logicalOlapTableSink(
+ logicalProject()
+ )
+ );
+ }
}
diff --git
a/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out
b/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out
index 2441607cf64..efaa9bc707a 100644
---
a/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out
+++
b/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out
@@ -27,3 +27,15 @@
1 2 3
10 2 12
+-- !delete_mow_without_light_delete_generated_column --
+2 20 21 200
+
+-- !delete_mow_without_light_delete_generated_key_column --
+2 21 20 200
+
+-- !delete_mow_without_light_delete_generated_column_with_not_null_value --
+2 20 21 200
+
+-- !delete_mow_without_light_delete_variant_generated_column --
+2 2026-06-08T11:00 order-2 {"city":"bj"} 200
+
diff --git
a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy
b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy
index 8cbe723d929..c38250d6b32 100644
---
a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy
+++
b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy
@@ -70,5 +70,101 @@ suite("test_generated_column_delete") {
) delete from test_par_gen_col_unique t1 using cte where t1.c=cte.c and
t1.b=cte.b"""
qt_delete_query_cte_select "select * from test_par_gen_col_unique order by
a,b,c"
+ multi_sql """
+ drop table if exists test_gen_col_mow_delete_partial_update;
+ create table test_gen_col_mow_delete_partial_update (
+ a int,
+ b int,
+ c int as (b + 1),
+ d int
+ )
+ unique key(a)
+ distributed by hash(a) buckets 1
+ properties (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "replication_num" = "1"
+ );
+ insert into test_gen_col_mow_delete_partial_update(a, b, d) values (1,
10, 100), (2, 20, 200);
+ """
+ sql "delete from test_gen_col_mow_delete_partial_update where a = 1;"
+ qt_delete_mow_without_light_delete_generated_column """
+ select * from test_gen_col_mow_delete_partial_update order by a;
+ """
+
+ multi_sql """
+ drop table if exists test_gen_key_mow_delete_partial_update;
+ create table test_gen_key_mow_delete_partial_update (
+ a int,
+ c int as (b + 1),
+ b int,
+ d int
+ )
+ unique key(a, c)
+ distributed by hash(a) buckets 1
+ properties (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "replication_num" = "1"
+ );
+ insert into test_gen_key_mow_delete_partial_update(a, b, d) values (1,
10, 100), (2, 20, 200);
+ """
+ sql "delete from test_gen_key_mow_delete_partial_update where a = 1;"
+ qt_delete_mow_without_light_delete_generated_key_column """
+ select * from test_gen_key_mow_delete_partial_update order by a;
+ """
+
+ multi_sql """
+ drop table if exists
test_gen_col_mow_delete_partial_update_not_null_value;
+ create table test_gen_col_mow_delete_partial_update_not_null_value (
+ a int,
+ b int,
+ c int as (b + 1),
+ d int not null
+ )
+ unique key(a)
+ distributed by hash(a) buckets 1
+ properties (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "replication_num" = "1"
+ );
+ insert into test_gen_col_mow_delete_partial_update_not_null_value(a,
b, d)
+ values (1, 10, 100), (2, 20, 200);
+ """
+ sql "delete from test_gen_col_mow_delete_partial_update_not_null_value
where a = 1;"
+ qt_delete_mow_without_light_delete_generated_column_with_not_null_value """
+ select * from test_gen_col_mow_delete_partial_update_not_null_value
order by a;
+ """
+
+ multi_sql """
+ drop table if exists test_gen_variant_mow_delete_partial_update;
+ create table test_gen_variant_mow_delete_partial_update (
+ id int not null,
+ create_time datetime not null,
+ order_no varchar(128) not null,
+ receive_address_detail varchar(1024) not null default "{}",
+ d int not null,
+ new_col variant as (receive_address_detail) null
+ )
+ unique key(id, create_time, order_no)
+ distributed by hash(order_no) buckets 1
+ properties (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "replication_num" = "1"
+ );
+ insert into test_gen_variant_mow_delete_partial_update(
+ id, create_time, order_no, receive_address_detail, d
+ ) values
+ (1, '2026-06-08 10:00:00', 'order-1', '{"city":"sh"}', 100),
+ (2, '2026-06-08 11:00:00', 'order-2', '{"city":"bj"}', 200);
+ """
+ sql "delete from test_gen_variant_mow_delete_partial_update where id = 1;"
+ qt_delete_mow_without_light_delete_variant_generated_column """
+ select id, create_time, order_no, receive_address_detail, d
+ from test_gen_variant_mow_delete_partial_update order by id;
+ """
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
