This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 755d35cd616 branch-3.0: [fix](compaction)Fix cluster key column
duplicated in value columns group #44610 (#44709)
755d35cd616 is described below
commit 755d35cd616df79856266716bf690bac6d6bf3b9
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Nov 28 16:33:55 2024 +0800
branch-3.0: [fix](compaction)Fix cluster key column duplicated in value
columns group #44610 (#44709)
Cherry-picked from #44610
---------
Co-authored-by: qiye <[email protected]>
---
be/src/olap/merger.cpp | 2 +-
.../test_index_compaction_unique_keys_arr.out | 67 ++++++++
.../test_index_compaction_unique_keys_arr.groovy | 191 ++++++++++++---------
3 files changed, 179 insertions(+), 81 deletions(-)
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index a79434551b5..bd7a1818051 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -201,7 +201,7 @@ void Merger::vertical_split_columns(const TabletSchema&
tablet_schema,
<< ", delete_sign_idx=" << delete_sign_idx;
// for duplicate no keys
if (!key_columns.empty()) {
- column_groups->emplace_back(std::move(key_columns));
+ column_groups->emplace_back(key_columns);
}
std::vector<uint32_t> value_columns;
diff --git
a/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
b/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
index 3edad07916c..a5406afb30d 100644
---
a/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
+++
b/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
@@ -67,3 +67,70 @@
2 bason bason hate pear 99
3 bason bason hate pear 99
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
+
+-- !sql --
+1 bason bason hate pear 99
+2 bason bason hate pear 99
+3 bason bason hate pear 99
diff --git
a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
index 013d5a55811..f7047735082 100644
---
a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
+++
b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
@@ -112,69 +112,22 @@ suite("test_index_compaction_unique_keys_arr",
"array_contains_inverted_index")
}
}
- boolean invertedIndexCompactionEnable = false
- boolean has_update_be_config = false
- try {
- String backend_id;
- backend_id = backendId_to_backendIP.keySet()[0]
- def (code, out, err) =
show_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id))
-
- logger.info("Show config: code=" + code + ", out=" + out + ", err=" +
err)
- assertEquals(code, 0)
- def configList = parseJson(out.trim())
- assert configList instanceof List
+ def run_test = { table_name ->
- for (Object ele in (List) configList) {
- assert ele instanceof List<String>
- if (((List<String>) ele)[0] == "inverted_index_compaction_enable")
{
- invertedIndexCompactionEnable =
Boolean.parseBoolean(((List<String>) ele)[2])
- logger.info("inverted_index_compaction_enable:
${((List<String>) ele)[2]}")
- }
- if (((List<String>) ele)[0] == "disable_auto_compaction") {
- disableAutoCompaction = Boolean.parseBoolean(((List<String>)
ele)[2])
- logger.info("disable_auto_compaction: ${((List<String>)
ele)[2]}")
- }
- }
- set_be_config.call("inverted_index_compaction_enable", "true")
- has_update_be_config = true
- // check updated config
- check_config.call("inverted_index_compaction_enable", "true");
+ sql """ INSERT INTO ${table_name} VALUES (1, "andy", "andy love
apple", 100); """
+ sql """ INSERT INTO ${table_name} VALUES (1, "bason", "bason hate
pear", 99); """
+ sql """ INSERT INTO ${table_name} VALUES (2, "andy", "andy love
apple", 100); """
+ sql """ INSERT INTO ${table_name} VALUES (2, "bason", "bason hate
pear", 99); """
+ sql """ INSERT INTO ${table_name} VALUES (3, "andy", "andy love
apple", 100); """
+ sql """ INSERT INTO ${table_name} VALUES (3, "bason", "bason hate
pear", 99); """
- sql """ DROP TABLE IF EXISTS ${tableName}; """
- sql """
- CREATE TABLE ${tableName} (
- `id` int(11) NULL,
- `name` varchar(255) NULL,
- `hobbies` text NULL,
- `score` int(11) NULL,
- index index_name (name) using inverted,
- index index_hobbies (hobbies) using inverted
properties("parser"="english"),
- index index_score (score) using inverted
- ) ENGINE=OLAP
- UNIQUE KEY(`id`)
- COMMENT 'OLAP'
- DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES (
- "replication_num" = "1",
- "disable_auto_compaction" = "true",
- "enable_unique_key_merge_on_write" = "true"
- );
- """
-
- sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple",
100); """
- sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate
pear", 99); """
- sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple",
100); """
- sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate
pear", 99); """
- sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple",
100); """
- sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate
pear", 99); """
-
- qt_sql """ select * from ${tableName} order by id, name, hobbies,
score """
- qt_sql """ select * from ${tableName} where name match "andy" order by
id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where hobbies match "pear" order
by id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where score < 100 order by id,
name, hobbies, score """
+ qt_sql """ select * from ${table_name} order by id, name, hobbies,
score """
+ qt_sql """ select * from ${table_name} where name match "andy" order
by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where hobbies match "pear"
order by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where score < 100 order by id,
name, hobbies, score """
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
- def tablets = sql_return_maparray """ show tablets from ${tableName};
"""
+ def tablets = sql_return_maparray """ show tablets from ${table_name};
"""
def dedup_tablets = deduplicate_tablets(tablets)
// In the p0 testing environment, there are no expected operations
such as scaling down BE (backend) services
@@ -189,7 +142,7 @@ suite("test_index_compaction_unique_keys_arr",
"array_contains_inverted_index")
int rowsetCount = get_rowset_count.call(tablets);
assert (rowsetCount == 7 * replicaNum)
- // trigger full compactions for all tablets in ${tableName}
+ // trigger full compactions for all tablets in ${table_name}
trigger_full_compaction_on_tablets.call(tablets)
// wait for full compaction done
@@ -203,23 +156,23 @@ suite("test_index_compaction_unique_keys_arr",
"array_contains_inverted_index")
assert (rowsetCount == 1 * replicaNum)
}
- qt_sql """ select * from ${tableName} order by id, name, hobbies,
score """
- qt_sql """ select * from ${tableName} where name match "andy" order by
id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where hobbies match "pear" order
by id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where score < 100 order by id,
name, hobbies, score """
+ qt_sql """ select * from ${table_name} order by id, name, hobbies,
score """
+ qt_sql """ select * from ${table_name} where name match "andy" order
by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where hobbies match "pear"
order by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where score < 100 order by id,
name, hobbies, score """
// insert more data and trigger full compaction again
- sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple",
100); """
- sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate
pear", 99); """
- sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple",
100); """
- sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate
pear", 99); """
- sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple",
100); """
- sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate
pear", 99); """
-
- qt_sql """ select * from ${tableName} order by id, name, hobbies,
score """
- qt_sql """ select * from ${tableName} where name match "andy" order by
id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where hobbies match "pear" order
by id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where score < 100 order by id,
name, hobbies, score """
+ sql """ INSERT INTO ${table_name} VALUES (1, "andy", "andy love
apple", 100); """
+ sql """ INSERT INTO ${table_name} VALUES (1, "bason", "bason hate
pear", 99); """
+ sql """ INSERT INTO ${table_name} VALUES (2, "andy", "andy love
apple", 100); """
+ sql """ INSERT INTO ${table_name} VALUES (2, "bason", "bason hate
pear", 99); """
+ sql """ INSERT INTO ${table_name} VALUES (3, "andy", "andy love
apple", 100); """
+ sql """ INSERT INTO ${table_name} VALUES (3, "bason", "bason hate
pear", 99); """
+
+ qt_sql """ select * from ${table_name} order by id, name, hobbies,
score """
+ qt_sql """ select * from ${table_name} where name match "andy" order
by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where hobbies match "pear"
order by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where score < 100 order by id,
name, hobbies, score """
rowsetCount = get_rowset_count.call(tablets);
if (isCloudMode) {
@@ -228,7 +181,7 @@ suite("test_index_compaction_unique_keys_arr",
"array_contains_inverted_index")
assert (rowsetCount == 7 * replicaNum)
}
- // trigger full compactions for all tablets in ${tableName}
+ // trigger full compactions for all tablets in ${table_name}
trigger_full_compaction_on_tablets.call(tablets)
// wait for full compaction done
@@ -242,11 +195,89 @@ suite("test_index_compaction_unique_keys_arr",
"array_contains_inverted_index")
assert (rowsetCount == 1 * replicaNum)
}
- qt_sql """ select * from ${tableName} order by id, name, hobbies,
score """
- qt_sql """ select * from ${tableName} where name match "andy" order by
id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where hobbies match "pear" order
by id, name, hobbies, score """
- qt_sql """ select * from ${tableName} where score < 100 order by id,
name, hobbies, score """
+ qt_sql """ select * from ${table_name} order by id, name, hobbies,
score """
+ qt_sql """ select * from ${table_name} where name match "andy" order
by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where hobbies match "pear"
order by id, name, hobbies, score """
+ qt_sql """ select * from ${table_name} where score < 100 order by id,
name, hobbies, score """
+ }
+ boolean invertedIndexCompactionEnable = false
+ boolean has_update_be_config = false
+ try {
+ String backend_id;
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) =
show_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id))
+
+ logger.info("Show config: code=" + code + ", out=" + out + ", err=" +
err)
+ assertEquals(code, 0)
+ def configList = parseJson(out.trim())
+ assert configList instanceof List
+
+ for (Object ele in (List) configList) {
+ assert ele instanceof List<String>
+ if (((List<String>) ele)[0] == "inverted_index_compaction_enable")
{
+ invertedIndexCompactionEnable =
Boolean.parseBoolean(((List<String>) ele)[2])
+ logger.info("inverted_index_compaction_enable:
${((List<String>) ele)[2]}")
+ }
+ if (((List<String>) ele)[0] == "disable_auto_compaction") {
+ disableAutoCompaction = Boolean.parseBoolean(((List<String>)
ele)[2])
+ logger.info("disable_auto_compaction: ${((List<String>)
ele)[2]}")
+ }
+ }
+ set_be_config.call("inverted_index_compaction_enable", "true")
+ has_update_be_config = true
+ // check updated config
+ check_config.call("inverted_index_compaction_enable", "true");
+
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
+ sql """
+ CREATE TABLE ${tableName} (
+ `id` int(11) NULL,
+ `name` varchar(255) NULL,
+ `hobbies` text NULL,
+ `score` int(11) NULL,
+ index index_name (name) using inverted,
+ index index_hobbies (hobbies) using inverted
properties("parser"="english"),
+ index index_score (score) using inverted
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+
+ run_test.call(tableName)
+
+ // cluster key is not supported in cloud mode at branch-3.0
+ if (!isCloudMode) {
+ tableName = "test_index_compaction_unique_keys_arr_cluster_key"
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
+ sql """
+ CREATE TABLE ${tableName} (
+ `id` int(11) NULL,
+ `name` varchar(255) NULL,
+ `hobbies` text NULL,
+ `score` int(11) NULL,
+ index index_name (name) using inverted,
+ index index_hobbies (hobbies) using inverted
properties("parser"="english"),
+ index index_score (score) using inverted
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ CLUSTER BY (`score`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+ run_test.call(tableName)
+ }
} finally {
if (has_update_be_config) {
set_be_config.call("inverted_index_compaction_enable",
invertedIndexCompactionEnable.toString())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
