This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new f42ebf277ce branch-4.1: [feature](fe) Support ANN indexes on MoW
tables (#67155)
f42ebf277ce is described below
commit f42ebf277cee24f8b00a2e012c028cb33f6f36a2
Author: Jack <[email protected]>
AuthorDate: Thu Aug 27 14:28:35 2026 +0800
branch-4.1: [feature](fe) Support ANN indexes on MoW tables (#67155)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #61994
Problem Summary:
This backports ANN index support for unique-key merge-on-write tables to
branch-4.1. FE validation previously restricted ANN indexes to DUP_KEYS
tables even though merge-on-write scans filter deleted rows before ANN
candidates are evaluated. The change permits ANN indexes only for
UNIQUE_KEYS tables with merge-on-write enabled, preserves rejection for
merge-on-read tables, and covers both the Nereids and legacy branch-4.1
CREATE INDEX validation paths.
### Release note
Support ANN indexes on unique-key merge-on-write tables.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Verification:
-
`ORC_FORMAT_URL=http://archive.apache.org/dist/orc/orc-format-1.0.0/orc-format-1.0.0.tar.gz
./build.sh --be --fe -j 160`
- `./run-fe-ut.sh --run
org.apache.doris.nereids.trees.plans.commands.IndexDefinitionTest,org.apache.doris.alter.SchemaChangeHandlerTest`
- `./run-regression-test.sh --run -d ann_index_p0 -s
ann_index_on_mow,build_ann_index_test,create_ann_index_test,create_tbl_with_ann_index_test
-parallel 4`
- Behavior changed:
- [ ] No.
- [x] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
Co-authored-by: ivin <[email protected]>
---
.../apache/doris/alter/SchemaChangeHandler.java | 8 ++-
.../java/org/apache/doris/analysis/IndexDef.java | 7 ++-
.../trees/plans/commands/info/IndexDefinition.java | 7 ++-
.../data/ann_index_p0/ann_index_on_mow.out | 31 +++++++++++
.../suites/ann_index_p0/ann_index_on_mow.groovy | 64 ++++++++++++++++++++++
.../ann_index_p0/build_ann_index_test.groovy | 45 +++++++++++++++
.../ann_index_p0/create_ann_index_test.groovy | 63 ++++++++++++++-------
.../create_tbl_with_ann_index_test.groovy | 22 ++++++++
8 files changed, 221 insertions(+), 26 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index a354af64741..c038fcd0e24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2806,8 +2806,12 @@ public class SchemaChangeHandler extends AlterHandler {
}
if (indexDef.isAnnIndex()) {
- if (olapTable.getKeysType() != KeysType.DUP_KEYS) {
- throw new AnalysisException("ANN index can only be built on
table with DUP_KEYS");
+ if (olapTable.getKeysType() != KeysType.DUP_KEYS
+ && !(olapTable.getKeysType() == KeysType.UNIQUE_KEYS
+ && olapTable.getEnableUniqueKeyMergeOnWrite())) {
+ throw new AnalysisException(
+ "ANN index can only be built on table with DUP_KEYS or
UNIQUE_KEYS"
+ + " with merge-on-write enabled");
}
AnnIndexPropertiesChecker.checkProperties(indexDef.getProperties());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index d64b65bc64a..abcbe848582 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -255,8 +255,11 @@ public class IndexDef {
if (!itemType.isFloatingPointType()) {
throw new AnalysisException("ANN index column item type must
be float type");
}
- if (keysType != KeysType.DUP_KEYS) {
- throw new AnalysisException("ANN index can only be used in
DUP_KEYS table");
+ if (keysType != KeysType.DUP_KEYS
+ && !(keysType == KeysType.UNIQUE_KEYS &&
enableUniqueKeyMergeOnWrite)) {
+ throw new AnalysisException(
+ "ANN index can only be used in DUP_KEYS table or
UNIQUE_KEYS table with"
+ + " merge-on-write enabled");
}
if (invertedIndexFileStorageFormat ==
TInvertedIndexFileStorageFormat.V1) {
throw new AnalysisException("ANN index is not supported in
index format V1");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
index 1d151801f92..8ab371db208 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
@@ -148,8 +148,11 @@ public class IndexDefinition {
if (!itemType.isFloatType()) {
throw new AnalysisException("ANN index column item type must
be float type, invalid index: " + name);
}
- if (keysType != KeysType.DUP_KEYS) {
- throw new AnalysisException("ANN index can only be used in
DUP_KEYS table");
+ if (keysType != KeysType.DUP_KEYS
+ && !(keysType == KeysType.UNIQUE_KEYS &&
enableUniqueKeyMergeOnWrite)) {
+ throw new AnalysisException(
+ "ANN index can only be used in DUP_KEYS table or
UNIQUE_KEYS table with"
+ + " merge-on-write enabled");
}
return;
}
diff --git a/regression-test/data/ann_index_p0/ann_index_on_mow.out
b/regression-test/data/ann_index_p0/ann_index_on_mow.out
new file mode 100644
index 00000000000..d0e14b53133
--- /dev/null
+++ b/regression-test/data/ann_index_p0/ann_index_on_mow.out
@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql_1 --
+1 [1, 2, 3] 11
+2 [4, 5, 6] 22
+3 [7, 8, 9] 33
+
+-- !sql_2 --
+1 0.0
+
+-- !sql_3 --
+1 [10, 20, 30] 111
+2 [40, 50, 60] 222
+3 [70, 80, 90] 333
+
+-- !sql_4 --
+1 0.0
+
+-- !sql_5 --
+1 [10, 20, 30] 111
+2 [40, 50, 60] 222
+3 [70, 80, 90] 333
+
+-- !sql_6 --
+
+-- !sql_7 --
+1 [10, 20, 30] \N 1 4
+2 [40, 50, 60] \N 1 4
+3 [70, 80, 90] \N 1 4
+
+-- !sql_8 --
+
diff --git a/regression-test/suites/ann_index_p0/ann_index_on_mow.groovy
b/regression-test/suites/ann_index_p0/ann_index_on_mow.groovy
new file mode 100644
index 00000000000..a14a116c533
--- /dev/null
+++ b/regression-test/suites/ann_index_p0/ann_index_on_mow.groovy
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("ann_index_on_mow") {
+ sql "drop table if exists ann_index_on_mow"
+ test {
+ sql """
+ CREATE TABLE ann_index_on_mow (
+ id INT NOT NULL COMMENT "",
+ vec ARRAY<FLOAT> NOT NULL COMMENT "",
+ value INT NULL COMMENT "",
+ INDEX ann_idx (vec) USING ANN PROPERTIES(
+ "index_type" = "hnsw",
+ "metric_type" = "l2_distance",
+ "dim" = "3"
+ )
+ ) ENGINE=OLAP
+ UNIQUE KEY(id) COMMENT "OLAP"
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+ }
+
+ sql "insert into ann_index_on_mow values (1, [1.0, 2.0, 3.0], 11),(2,
[4.0, 5.0, 6.0], 22),(3, [7.0, 8.0, 9.0], 33)"
+
+ qt_sql_1 "select * from ann_index_on_mow order by id"
+
+ qt_sql_2 "select id, l2_distance_approximate(vec, [1.0, 2.0, 3.0]) as dist
from ann_index_on_mow order by dist limit 1;"
+
+ sql "insert into ann_index_on_mow values (1, [10.0, 20.0, 30.0], 111),(2,
[40.0, 50.0, 60.0], 222),(3, [70.0, 80.0, 90.0], 333)"
+
+ qt_sql_3 "select * from ann_index_on_mow order by id"
+
+ qt_sql_4 "select id, l2_distance_approximate(vec, [10.0, 20.0, 30.0]) as
dist from ann_index_on_mow order by dist limit 1;"
+
+ qt_sql_5 "select * from ann_index_on_mow order by id"
+
+ sql "insert into ann_index_on_mow (id, vec, __DORIS_DELETE_SIGN__) values
(1, [10.0, 20.0, 30.0], 1),(2, [40.0, 50.0, 60.0], 1),(3, [70.0, 80.0, 90.0],
1);"
+
+ qt_sql_6 "select * from ann_index_on_mow order by id"
+
+ sql "set show_hidden_columns=true;"
+ qt_sql_7 "select * from ann_index_on_mow order by id"
+ sql "set show_hidden_columns=false;"
+
+ qt_sql_8 "select id, l2_distance_approximate(vec, [10.0, 20.0, 30.0]) as
dist from ann_index_on_mow order by dist limit 1;"
+}
diff --git a/regression-test/suites/ann_index_p0/build_ann_index_test.groovy
b/regression-test/suites/ann_index_p0/build_ann_index_test.groovy
index e8de0d3d2d1..26101a60950 100644
--- a/regression-test/suites/ann_index_p0/build_ann_index_test.groovy
+++ b/regression-test/suites/ann_index_p0/build_ann_index_test.groovy
@@ -109,4 +109,49 @@ suite("build_ann_index_test") {
// BUILD INDEX
sql "BUILD INDEX idx_test_ann ON table_build_ann_index_test;"
wait_for_last_build_index_on_table_finish(tableName, timeout)
+
+
+ // case 2: create and build ann index on mow table
+ sql "drop table if exists table_build_ann_index_on_mow;"
+
+ sql """
+ CREATE TABLE `table_build_ann_index_on_mow` (
+ `id` int NOT NULL COMMENT "",
+ `embedding` array<float> NOT NULL COMMENT ""
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`) COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+
+ sql """
+ INSERT INTO table_build_ann_index_on_mow (id, embedding) VALUES
+ (0, [39.906116, 10.495334, 54.08394, 88.67262, 55.243687, 10.162686,
36.335983, 38.684258]),
+ (1, [62.759315, 97.15586, 25.832521, 39.604908, 88.76715, 72.64085,
9.688437, 17.721428]),
+ (2, [15.447449, 59.7771, 65.54516, 12.973712, 99.685135, 72.080734,
85.71118, 99.35976]),
+ (3, [72.26747, 46.42257, 32.368374, 80.50209, 5.777631, 98.803314,
7.0915947, 68.62693]),
+ (4, [22.098177, 74.10027, 63.634556, 4.710955, 12.405106, 79.39356,
63.014366, 68.67834]),
+ (5, [27.53003, 72.1106, 50.891026, 38.459953, 68.30715, 20.610682,
94.806274, 45.181377]),
+ (6, [77.73215, 64.42907, 71.50025, 43.85641, 94.42648, 50.04773,
65.12575, 68.58207]),
+ (7, [2.1537063, 82.667885, 16.171143, 71.126656, 5.335274, 40.286068,
11.943586, 3.69409]),
+ (8, [54.435013, 56.800594, 59.335514, 55.829235, 85.46627, 33.388138,
11.076194, 20.480877]),
+ (9, [76.197945, 60.623528, 84.229805, 31.652937, 71.82595, 48.04684,
71.29212, 30.282396]);
+ """
+
+ // CREATE INDEX
+ sql """
+ CREATE INDEX idx_test_ann ON table_build_ann_index_on_mow(`embedding`)
USING ANN PROPERTIES(
+ "index_type"="hnsw",
+ "metric_type"="l2_distance",
+ "dim"="8"
+ );
+ """
+ wait_for_latest_op_on_table_finish("table_build_ann_index_on_mow", timeout)
+
+ // BUILD INDEX
+ sql "BUILD INDEX idx_test_ann ON table_build_ann_index_on_mow;"
+ wait_for_last_build_index_on_table_finish("table_build_ann_index_on_mow",
timeout)
}
diff --git a/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
b/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
index 17f542de168..3e32584a378 100644
--- a/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
+++ b/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
@@ -39,6 +39,31 @@ suite("create_ann_index_test") {
);
"""
+ sql "drop table if exists tbl_mor_without_ann"
+ sql """
+ CREATE TABLE `tbl_mor_without_ann` (
+ `id` int NOT NULL COMMENT "",
+ `embedding` array<float> NOT NULL COMMENT ""
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`) COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+
+ test {
+ sql """
+ CREATE INDEX idx_test_ann_on_mor ON tbl_mor_without_ann(`embedding`)
USING ANN PROPERTIES(
+ "index_type"="hnsw",
+ "metric_type"="l2_distance",
+ "dim"="1"
+ );
+ """
+ exception "ANN index can only be built on table with DUP_KEYS or
UNIQUE_KEYS with merge-on-write enabled"
+ }
+
// Test cases for creating tables with ANN indexes
// 1. Case for nullable column
@@ -299,26 +324,24 @@ suite("create_ann_index_test") {
);
"""
- sql "drop table if exists tbl_ann_unique_key"
- test {
- sql """
- CREATE TABLE tbl_ann_unique_key (
- id INT NOT NULL COMMENT "",
- embedding ARRAY<FLOAT> NOT NULL COMMENT "",
- INDEX idx_test_ann (`embedding`) USING ANN PROPERTIES(
- "index_type"="hnsw",
- "metric_type"="inner_product",
- "dim"="128"
- )
- ) ENGINE=OLAP
- UNIQUE KEY(id) COMMENT "OLAP"
- DISTRIBUTED BY HASH(id) BUCKETS 2
- PROPERTIES (
- "replication_num" = "1"
- );
- """
- exception "ANN index can only be used in DUP_KEYS table"
- }
+ sql "drop table if exists tbl_ann_on_mow"
+ sql """
+ CREATE TABLE tbl_ann_on_mow (
+ id INT NOT NULL COMMENT "",
+ embedding ARRAY<FLOAT> NOT NULL COMMENT "",
+ INDEX idx_test_ann (`embedding`) USING ANN PROPERTIES(
+ "index_type"="hnsw",
+ "metric_type"="inner_product",
+ "dim"="128"
+ )
+ ) ENGINE=OLAP
+ UNIQUE KEY(id) COMMENT "OLAP"
+ DISTRIBUTED BY HASH(id) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
sql "drop table if exists tbl_efconstruction"
sql """
diff --git
a/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy
b/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy
index 35725cd4b46..03c897d2760 100644
--- a/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy
+++ b/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy
@@ -37,6 +37,28 @@ suite("create_tbl_with_ann_index_test") {
"""
}
+ sql "drop table if exists ann_tbl_mor"
+ test {
+ sql """
+ CREATE TABLE ann_tbl_mor (
+ id INT NOT NULL COMMENT "",
+ vec ARRAY<FLOAT> NOT NULL COMMENT "",
+ INDEX ann_idx_mor (vec) USING ANN PROPERTIES(
+ "index_type" = "hnsw",
+ "metric_type" = "l2_distance",
+ "dim" = "128"
+ )
+ ) ENGINE=OLAP
+ UNIQUE KEY(id) COMMENT "OLAP"
+ DISTRIBUTED BY HASH(id) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+ exception "ANN index can only be used in DUP_KEYS table or UNIQUE_KEYS
table with merge-on-write enabled"
+ }
+
sql "drop table if exists ann_tbl2"
test {
sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
