This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 3c290348901 [opt](inverted index) default index format uses v3 (#57252)
3c290348901 is described below
commit 3c290348901aa19d77c6790442fed316c8edb0d7
Author: zzzxl <[email protected]>
AuthorDate: Thu Oct 23 14:58:32 2025 +0800
[opt](inverted index) default index format uses v3 (#57252)
https://github.com/apache/doris/pull/57140
---
be/src/olap/tablet_meta.cpp | 2 +-
.../main/java/org/apache/doris/common/Config.java | 4 +-
.../cloud/datasource/CloudInternalCatalog.java | 6 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 12 +--
.../apache/doris/common/PropertyAnalyzerTest.java | 90 +++++++++++----------
.../storage_format/test_storage_format_default.out | 4 +
.../data/query_p0/system/test_table_properties.out | 12 +--
.../test_show_create_table_and_views_nereids.out | 8 +-
.../test_index_change_on_renamed_column.groovy | 2 +-
...est_cumulative_compaction_with_format_v2.groovy | 2 +-
.../test_mor_table_with_format_v2.groovy | 4 +-
.../test_mow_table_with_format_v2.groovy | 2 +-
.../test_storage_format_default.groovy | 91 ++++++++++++++++++++++
.../predefine/test_multi_index_file.groovy | 2 +-
14 files changed, 171 insertions(+), 70 deletions(-)
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index bd23ed3f472..652b8bf6c25 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -238,7 +238,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
break;
default:
-
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
+
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
break;
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 3b707240aab..9597815fcc4 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3038,9 +3038,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true, description = {
"倒排索引默认存储格式",
- "Default storage format of inverted index, the default value is
V1."
+ "Default storage format of inverted index, the default value is
V3."
})
- public static String inverted_index_storage_format = "V2";
+ public static String inverted_index_storage_format = "V3";
@ConfField(mutable = true, masterOnly = true, description = {
"是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 8bd43890ba5..a8d61b39c27 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -350,10 +350,10 @@ public class CloudInternalCatalog extends InternalCatalog
{
} else if (invertedIndexFileStorageFormat ==
TInvertedIndexFileStorageFormat.DEFAULT) {
if
(Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1);
- } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V3")) {
-
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3);
- } else {
+ } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2);
+ } else {
+
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3);
}
} else {
throw new DdlException("invalid inverted index storage
format");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 3da280320f9..61ef3ed4143 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -1194,10 +1194,10 @@ public class PropertyAnalyzer {
} else {
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
return TInvertedIndexFileStorageFormat.V1;
- } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V3")) {
- return TInvertedIndexFileStorageFormat.V3;
- } else {
+ } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
return TInvertedIndexFileStorageFormat.V2;
+ } else {
+ return TInvertedIndexFileStorageFormat.V3;
}
}
@@ -1210,10 +1210,10 @@ public class PropertyAnalyzer {
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default"))
{
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
return TInvertedIndexFileStorageFormat.V1;
- } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V3")) {
- return TInvertedIndexFileStorageFormat.V3;
- } else {
+ } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
return TInvertedIndexFileStorageFormat.V2;
+ } else {
+ return TInvertedIndexFileStorageFormat.V3;
}
} else {
throw new AnalysisException("unknown inverted index storage
format: " + invertedIndexFileStorageFormat);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index ddd813df376..926d803f931 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -289,49 +289,55 @@ public class PropertyAnalyzerTest {
@Test
public void testAnalyzeInvertedIndexFileStorageFormat() throws
AnalysisException {
- TInvertedIndexFileStorageFormat result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result);
-
- Config.inverted_index_storage_format = "V1";
- result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new
HashMap<>());
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
-
- Map<String, String> propertiesWithV1 = new HashMap<>();
-
propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v1");
- result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
-
- Map<String, String> propertiesWithV2 = new HashMap<>();
-
propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v2");
- result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV2);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result);
-
- Map<String, String> propertiesWithV3 = new HashMap<>();
-
propertiesWithV3.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v3");
- result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result);
-
- Config.inverted_index_storage_format = "V1";
- Map<String, String> propertiesWithDefaultV1 = new HashMap<>();
-
propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"default");
- result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
-
- Config.inverted_index_storage_format = "V2";
- Map<String, String> propertiesWithDefaultV2 = new HashMap<>();
-
propertiesWithDefaultV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"default");
- result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV2);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result);
-
- Map<String, String> propertiesWithUnknown = new HashMap<>();
-
propertiesWithUnknown.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"unknown_format");
+ String originFormat = Config.inverted_index_storage_format;
try {
-
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithUnknown);
- Assertions.fail("Expected AnalysisException was not thrown");
- } catch (AnalysisException e) {
- Assertions.assertEquals(
- "errCode = 2, detailMessage = unknown inverted index
storage format: unknown_format",
- e.getMessage());
+ Config.inverted_index_storage_format = "V3";
+ TInvertedIndexFileStorageFormat result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3,
result);
+
+ Config.inverted_index_storage_format = "V1";
+ result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new HashMap<>());
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1,
result);
+
+ Map<String, String> propertiesWithV1 = new HashMap<>();
+
propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v1");
+ result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1,
result);
+
+ Map<String, String> propertiesWithV2 = new HashMap<>();
+
propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v2");
+ result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV2);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2,
result);
+
+ Map<String, String> propertiesWithV3 = new HashMap<>();
+
propertiesWithV3.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v3");
+ result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3,
result);
+
+ Config.inverted_index_storage_format = "V1";
+ Map<String, String> propertiesWithDefaultV1 = new HashMap<>();
+
propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"default");
+ result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1,
result);
+
+ Config.inverted_index_storage_format = "V2";
+ Map<String, String> propertiesWithDefaultV2 = new HashMap<>();
+
propertiesWithDefaultV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"default");
+ result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV2);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2,
result);
+
+ Map<String, String> propertiesWithUnknown = new HashMap<>();
+
propertiesWithUnknown.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"unknown_format");
+ try {
+
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithUnknown);
+ Assertions.fail("Expected AnalysisException was not thrown");
+ } catch (AnalysisException e) {
+ Assertions.assertEquals(
+ "errCode = 2, detailMessage = unknown inverted index
storage format: unknown_format",
+ e.getMessage());
+ }
+ } finally {
+ Config.inverted_index_storage_format = originFormat;
}
}
diff --git
a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_default.out
b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_default.out
new file mode 100644
index 00000000000..71cd5aaf2b0
--- /dev/null
+++
b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_default.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+863
+
diff --git a/regression-test/data/query_p0/system/test_table_properties.out
b/regression-test/data/query_p0/system/test_table_properties.out
index 598a4ea0ea5..8d32e2f55db 100644
--- a/regression-test/data/query_p0/system/test_table_properties.out
+++ b/regression-test/data/query_p0/system/test_table_properties.out
@@ -20,7 +20,7 @@ internal test_table_properties_db duplicate_table
file_cache_ttl_seconds 0
internal test_table_properties_db duplicate_table
group_commit_data_bytes 134217728
internal test_table_properties_db duplicate_table
group_commit_interval_ms 10000
internal test_table_properties_db duplicate_table in_memory
false
-internal test_table_properties_db duplicate_table
inverted_index_storage_format V2
+internal test_table_properties_db duplicate_table
inverted_index_storage_format V3
internal test_table_properties_db duplicate_table is_being_synced
false
internal test_table_properties_db duplicate_table
light_schema_change true
internal test_table_properties_db duplicate_table
min_load_replica_num -1
@@ -55,7 +55,7 @@ internal test_table_properties_db listtable
file_cache_ttl_seconds 0
internal test_table_properties_db listtable
group_commit_data_bytes 134217728
internal test_table_properties_db listtable
group_commit_interval_ms 10000
internal test_table_properties_db listtable in_memory
false
-internal test_table_properties_db listtable
inverted_index_storage_format V2
+internal test_table_properties_db listtable
inverted_index_storage_format V3
internal test_table_properties_db listtable is_being_synced
false
internal test_table_properties_db listtable
light_schema_change true
internal test_table_properties_db listtable
min_load_replica_num -1
@@ -90,7 +90,7 @@ internal test_table_properties_db unique_table
file_cache_ttl_seconds 0
internal test_table_properties_db unique_table
group_commit_data_bytes 134217728
internal test_table_properties_db unique_table
group_commit_interval_ms 10000
internal test_table_properties_db unique_table in_memory
false
-internal test_table_properties_db unique_table
inverted_index_storage_format V2
+internal test_table_properties_db unique_table
inverted_index_storage_format V3
internal test_table_properties_db unique_table is_being_synced
false
internal test_table_properties_db unique_table
light_schema_change true
internal test_table_properties_db unique_table
min_load_replica_num -1
@@ -127,7 +127,7 @@ internal test_table_properties_db duplicate_table
file_cache_ttl_seconds 0
internal test_table_properties_db duplicate_table
group_commit_data_bytes 134217728
internal test_table_properties_db duplicate_table
group_commit_interval_ms 10000
internal test_table_properties_db duplicate_table in_memory
false
-internal test_table_properties_db duplicate_table
inverted_index_storage_format V2
+internal test_table_properties_db duplicate_table
inverted_index_storage_format V3
internal test_table_properties_db duplicate_table is_being_synced
false
internal test_table_properties_db duplicate_table
light_schema_change true
internal test_table_properties_db duplicate_table
min_load_replica_num -1
@@ -162,7 +162,7 @@ internal test_table_properties_db unique_table
file_cache_ttl_seconds 0
internal test_table_properties_db unique_table
group_commit_data_bytes 134217728
internal test_table_properties_db unique_table
group_commit_interval_ms 10000
internal test_table_properties_db unique_table in_memory
false
-internal test_table_properties_db unique_table
inverted_index_storage_format V2
+internal test_table_properties_db unique_table
inverted_index_storage_format V3
internal test_table_properties_db unique_table is_being_synced
false
internal test_table_properties_db unique_table
light_schema_change true
internal test_table_properties_db unique_table
min_load_replica_num -1
@@ -201,7 +201,7 @@ internal test_table_properties_db duplicate_table
file_cache_ttl_seconds 0
internal test_table_properties_db duplicate_table
group_commit_data_bytes 134217728
internal test_table_properties_db duplicate_table
group_commit_interval_ms 10000
internal test_table_properties_db duplicate_table in_memory
false
-internal test_table_properties_db duplicate_table
inverted_index_storage_format V2
+internal test_table_properties_db duplicate_table
inverted_index_storage_format V3
internal test_table_properties_db duplicate_table is_being_synced
false
internal test_table_properties_db duplicate_table
light_schema_change true
internal test_table_properties_db duplicate_table
min_load_replica_num -1
diff --git
a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
index fe47469aeb5..7929b470514 100644
--- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
+++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !show --
-show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n
`good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX
index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX
index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES [("-17014118346046923 [...]
+show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n
`good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX
index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX
index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES [("-17014118346046923 [...]
-- !select --
1 1 30
@@ -36,11 +36,11 @@ show_create_table_and_views_nereids_view CREATE VIEW
`show_create_table_and_view
300 1
-- !show --
-show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n
`good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX
index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX
index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES [("-17014118346046923 [...]
+show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n
`good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX
index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX
index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES [("-17014118346046923 [...]
-- !show --
-show_create_table_and_views_nereids_like CREATE TABLE
`show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n
`good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX
index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX
index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES [("-1701411834604692317 [...]
+show_create_table_and_views_nereids_like CREATE TABLE
`show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n
`good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX
index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX
index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES [("-1701411834604692317 [...]
-- !show --
-show_create_table_and_views_nereids_like_with_rollup CREATE TABLE
`show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint
NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT
"0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index
comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test
index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`,
`good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [...]
+show_create_table_and_views_nereids_like_with_rollup CREATE TABLE
`show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint
NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT
"0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index
comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test
index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`,
`good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [...]
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy
index c685f2d1215..c495e207a0c 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy
@@ -97,7 +97,7 @@ suite("test_index_change_on_renamed_column") {
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- check_nested_index_file(ip, port, tablet_id, 3, 1, "V2")
+ check_nested_index_file(ip, port, tablet_id, 3, 1, "V3")
// drop inverted index on renamed column
sql """ alter table ${tableName} drop index idx_s; """
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
index 6af721010ad..24bcfb46ef0 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
@@ -213,7 +213,7 @@ suite("test_cumulative_compaction_with_format_v2",
"inverted_index_format_v2") {
assertTrue(hasCompactedRowsets, "Should have compacted rowsets
starting from version 2")
int segment_count = calc_segment_count(tablet)
logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3,
"V2")
+ check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3,
"V3")
}
int segmentsCount = 0
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
index 61aadbcd0bb..cb438df4142 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
@@ -175,9 +175,9 @@ suite("test_mor_table_with_format_v2",
"inverted_index_format_v2") {
String port = backendId_to_backendHttpPort.get(backend_id)
// after compaction, there are 1 rwoset in local mode and 2
rowsets in cloud mode.
if (isCloudMode()) {
- check_nested_index_file(ip, port, tablet_id, 2, 3, "V2")
+ check_nested_index_file(ip, port, tablet_id, 2, 3, "V3")
} else {
- check_nested_index_file(ip, port, tablet_id, 1, 3, "V2")
+ check_nested_index_file(ip, port, tablet_id, 1, 3, "V3")
}
}
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
index 7fa1a2f93d1..7d99de66f94 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
@@ -211,7 +211,7 @@ suite("test_mow_table_with_format_v2",
"inverted_index_format_v2") {
boolean hasCompactedRowsets = rowsets.any { it.contains("[2-") }
assertTrue(hasBaseRowset, "Should have base rowset [0-1]")
assertTrue(hasCompactedRowsets, "Should have compacted rowsets
starting from version 2")
- check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3,
"V2")
+ check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3,
"V3")
}
int segmentsCount = 0
diff --git
a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy
b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy
new file mode 100644
index 00000000000..54d55aa73cb
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_storage_format_default", "p0") {
+ def testTable = "httplogs_dup_default"
+
+ def create_httplogs_dup_table = { tableName ->
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `@timestamp` int(11) NULL,
+ `clientip` varchar(20) NULL,
+ `request` string NULL,
+ `status` int(11) NULL,
+ `size` int(11) NULL,
+ INDEX size_idx (`size`) USING INVERTED COMMENT '',
+ INDEX status_idx (`status`) USING INVERTED COMMENT '',
+ INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
+ INDEX request_idx (`request`) USING INVERTED
PROPERTIES("support_phrase" = "true", "parser" = "english", "lower_case" =
"true") COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`@timestamp`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 2
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true"
+ );
+ """
+ }
+
+ def load_httplogs_data = { tableName, label, readFlag, formatFlag,
fileName, ignoreFailure = false,
+ expectedSuccRows = -1 ->
+ streamLoad {
+ table "${tableName}"
+
+ set 'label', label + "_" + UUID.randomUUID().toString()
+ set 'read_json_by_line', readFlag
+ set 'format', formatFlag
+ file fileName
+ time 10000
+ if (expectedSuccRows >= 0) {
+ set 'max_filter_ratio', '1'
+ }
+
+ check { result, exception, startTime, endTime ->
+ if (ignoreFailure && expectedSuccRows < 0) { return }
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ if (expectedSuccRows >= 0) {
+ assertEquals(json.NumberLoadedRows, expectedSuccRows)
+ } else {
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows +
json.NumberUnselectedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+ }
+
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ create_httplogs_dup_table.call(testTable)
+
+ def showCreate = sql """ SHOW CREATE TABLE ${testTable} """
+ assertTrue(showCreate.size() > 0)
+
assertTrue(showCreate[0][1].contains("\"inverted_index_storage_format\" =
\"V3\""))
+
+ load_httplogs_data.call(testTable, 'label', 'true', 'json',
'documents-1000.json')
+ sql "sync"
+
+ qt_sql("select COUNT(*) from ${testTable} where request match
'images'")
+ } finally {
+ sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+}
\ No newline at end of file
diff --git
a/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy
b/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy
index 9e1b7344202..61512328f4e 100644
--- a/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy
+++ b/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy
@@ -41,5 +41,5 @@ suite("test_variant_multi_index_file", "p0, nonConcurrent"){
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- check_nested_index_file(ip, port, tablet_id, 2, 4, "V2")
+ check_nested_index_file(ip, port, tablet_id, 2, 4, "V3")
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
