This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d1070cb002e [fix](fe) block inverted index V1 creation in FE (#64522)
d1070cb002e is described below
commit d1070cb002ecb78af59a190bab49f8679b8841e9
Author: liangj777 <[email protected]>
AuthorDate: Mon Aug 24 10:47:56 2026 +0800
[fix](fe) block inverted index V1 creation in FE (#64522)
### What problem does this PR solve?
Problem Summary:
When users copy historical DDL statements that explicitly set inverted
index format to V1, new tables are silently created with V1. Inverted
index V1 does not support small file merging and bypasses the packed
file path, significantly amplifying PUT requests to object storage.
Observed impact: clusters with high write QPS are seeing300–400 PUT/s,
with 70–80% attributed to inverted index V1.
Block new inverted index V1 creation at the FE level:
- Explicitly specifying `inverted_index_storage_format = v1` on a new
indexnow returns an error immediately.
- Users are guided to use V2 (the default).
- Existing V1 tables remain fully compatible — metadata loading and
read/write
are unaffected. No forced migration.
Error message:Inverted index V1 is deprecated and no longer allowed for
new index creation.Please use inverted index V2.
Coverage
- `CREATE TABLE` with inline index definition
- `ALTER TABLE ADD INDEX`
- All other code paths that create a new inverted index
### Release note
---
.../main/java/org/apache/doris/common/Config.java | 5 +-
.../InvertedIndexStorageFormatValidator.java | 50 ++++++++++
.../java/org/apache/doris/common/ConfigTest.java | 20 ++++
.../src/main/java/org/apache/doris/DorisFE.java | 5 +
.../apache/doris/alter/SchemaChangeHandler.java | 7 ++
.../apache/doris/common/util/PropertyAnalyzer.java | 12 +--
.../apache/doris/common/PropertyAnalyzerTest.java | 12 ++-
.../fault_injection_p0/test_index_io_context.out | 93 ------------------
.../inverted_index_p0/test_index_multi_match.out | 96 ------------------
.../ann_index_p0/create_ann_index_test.groovy | 2 +-
.../test_mv_case/test_mv_case.groovy | 6 +-
.../test_special_buffer_before_nested.groovy | 2 +-
.../test_index_bkd_writer_fault_injection.groovy | 2 +-
...index_builder_drop_index_fault_injection.groovy | 1 -
...dex_compaction_exception_fault_injection.groovy | 2 +-
.../test_index_compaction_fault_injection.groovy | 4 +-
.../test_index_io_context.groovy | 35 +------
...st_skip_index_compaction_fault_injection.groovy | 27 ------
...inverted_index_exception_fault_injection.groovy | 2 +-
...st_build_index_exception_fault_injection.groovy | 2 +-
.../suites/function_p0/test_array_map.groovy | 2 +-
.../test_iot_auto_detect_fail.groovy | 4 +-
.../index_change/test_index_change_1.groovy | 75 ++------------
.../index_change/test_index_change_2.groovy | 84 ++--------------
.../index_change/test_index_change_3.groovy | 89 ++---------------
.../index_change/test_index_change_4.groovy | 88 ++---------------
.../index_change/test_index_change_5.groovy | 79 ++-------------
.../index_change/test_index_change_6.groovy | 99 ++-----------------
.../test_index_change_on_new_column.groovy | 50 +---------
.../test_index_compaction_dup_keys.groovy | 2 +-
.../test_index_compaction_null.groovy | 4 +-
.../test_index_compaction_unique_keys.groovy | 2 +-
...dex_compaction_with_multi_index_segments.groovy | 4 +-
.../test_index_change_format.groovy | 1 -
.../inverted_index_p0/load/test_insert.groovy | 4 +-
.../inverted_index_p0/load/test_spark_load.groovy | 2 -
.../inverted_index_p0/load/test_stream_load.groovy | 1 -
.../test_inverted_index_v1_deprecated.groovy | 80 +++++++++++++++
.../test_compound_reader_fault_injection.groovy | 2 +-
.../test_index_chinese_column.groovy | 2 -
.../test_index_multi_match.groovy | 42 --------
.../inverted_index_p0/test_index_rqg_bug8.groovy | 2 +-
.../test_inverted_index_file_size.groovy | 2 -
.../test_show_nested_index_file_http_action.groovy | 1 -
...sted_index_file_http_action_with_variant.groovy | 1 -
.../test_variant_index_format_v1.groovy | 108 ---------------------
.../nereids_arith_p0/topn/accept_null.groovy | 2 +-
.../show/test_nereids_show_build_index.groovy | 4 +-
.../query_p0/show/test_nereids_show_index.groovy | 4 +-
.../test_cloud_inverted_index_v1_show_data.groovy | 101 -------------------
.../test_array_contains_with_inverted_index.groovy | 5 +-
.../suites/variant_p0/mv/multi_slot.groovy | 2 +-
.../variant_p0/predefine/test_predefine_ddl.groovy | 12 ---
.../test_array_contains_with_inverted_index.groovy | 5 +-
.../test_array_contains_with_inverted_index.groovy | 5 +-
.../with_index/test_array_index_write.groovy | 5 +-
.../suites/variant_p0/with_index/var_index.groovy | 105 --------------------
57 files changed, 258 insertions(+), 1207 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 4fe558fe3e4..6ba1e477efc 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2822,8 +2822,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean fix_tablet_partition_id_eq_0 = false;
- @ConfField(mutable = true, masterOnly = true, description = "Default
storage format of inverted index, the "
- + "default value is V3.")
+ @ConfField(mutable = true, masterOnly = true,
+ callback =
InvertedIndexStorageFormatValidator.RuntimeConfigHandler.class,
+ description = "Default storage format of inverted index, the
default value is V3.")
public static String inverted_index_storage_format = "V3";
@ConfField(mutable = true, masterOnly = true, description = "Enable the
'delete predicate' for DELETE statements. "
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/common/InvertedIndexStorageFormatValidator.java
b/fe/fe-common/src/main/java/org/apache/doris/common/InvertedIndexStorageFormatValidator.java
new file mode 100644
index 00000000000..7b8809f04dd
--- /dev/null
+++
b/fe/fe-common/src/main/java/org/apache/doris/common/InvertedIndexStorageFormatValidator.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common;
+
+import java.lang.reflect.Field;
+
+public final class InvertedIndexStorageFormatValidator {
+ private InvertedIndexStorageFormatValidator() {
+ }
+
+ public static void rejectRuntimeV1(String confVal) throws ConfigException {
+ String normalizedConfVal = confVal.trim();
+ if ("V1".equalsIgnoreCase(normalizedConfVal)) {
+ throw new ConfigException("Inverted index V1 is deprecated and no
longer allowed"
+ + " for new index creation. Please use inverted index
V2.");
+ }
+ }
+
+ public static void rejectStartupV1(String confVal) throws ConfigException {
+ String normalizedConfVal = confVal.trim();
+ if ("V1".equalsIgnoreCase(normalizedConfVal)) {
+ throw new ConfigException(
+ "inverted_index_storage_format=V1 is no longer supported. "
+ + "Please update fe.conf (or fe_custom.conf): set
inverted_index_storage_format=V2.");
+ }
+ }
+
+ public static class RuntimeConfigHandler extends
ConfigBase.DefaultConfHandler {
+ @Override
+ public void handle(Field field, String confVal) throws Exception {
+ rejectRuntimeV1(confVal);
+ super.handle(field, confVal.trim());
+ }
+ }
+}
diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
index 09572c618c7..aa1819ee508 100644
--- a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
+++ b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
@@ -131,4 +131,24 @@ public class ConfigTest {
Assert.assertTrue(e.getMessage().contains("is not mutable"));
}
}
+
+ @Test
+ public void testRejectDeprecatedInvertedIndexV1WithWhitespace() throws
Exception {
+ String originFormat = Config.inverted_index_storage_format;
+ try {
+ ConfigBase.setMutableConfig("inverted_index_storage_format", "V2");
+ ConfigException dynamicException =
Assert.assertThrows(ConfigException.class,
+ () ->
ConfigBase.setMutableConfig("inverted_index_storage_format", " V1 "));
+ Assert.assertTrue(dynamicException.getMessage().contains("Inverted
index V1 is deprecated"));
+ Assert.assertEquals("V2", Config.inverted_index_storage_format);
+
+ Config.inverted_index_storage_format = "V2";
+ ConfigException startupException =
Assert.assertThrows(ConfigException.class,
+ () ->
InvertedIndexStorageFormatValidator.rejectStartupV1(" V1 "));
+
Assert.assertTrue(startupException.getMessage().contains("inverted_index_storage_format=V1"));
+ Assert.assertEquals("V2", Config.inverted_index_storage_format);
+ } finally {
+ Config.inverted_index_storage_format = originFormat;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
index dfd4f2e9404..f75f6e54f3f 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.CommandLineOptions;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.InvertedIndexStorageFormatValidator;
import org.apache.doris.common.LdapConfig;
import org.apache.doris.common.Log4jConfig;
import org.apache.doris.common.LogUtils;
@@ -149,6 +150,10 @@ public class DorisFE {
// Must init custom config after init config, separately.
// Because the path of custom config file is defined in fe.conf
config.initCustom(Config.custom_config_dir + "/fe_custom.conf");
+ // inverted_index_storage_format's runtime callback is not invoked
while parsing
+ // fe.conf/fe_custom.conf, so validate the loaded value here after
both files are loaded
+ // and merged, to reject a "V1" left over in the config files at
startup.
+
InvertedIndexStorageFormatValidator.rejectStartupV1(Config.inverted_index_storage_format);
// The command line value overrides fe.conf, so this can only run
once Config is loaded.
resolveLocalResourceGroup(commandLine);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index a1a0f0fcfce..705fd1c1f97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -3290,6 +3290,13 @@ public class SchemaChangeHandler extends AlterHandler {
AnnIndexPropertiesChecker.checkProperties(indexDef.getProperties());
}
+ if (indexDef.getIndexType() == IndexType.INVERTED
+ && olapTable.getInvertedIndexFileStorageFormat() ==
TInvertedIndexFileStorageFormat.V1) {
+ throw new DdlException("Inverted index V1 is deprecated and no
longer allowed for new index creation."
+ + " Upgrading inverted_index_storage_format via ALTER
TABLE is not supported;"
+ + " recreate the table with inverted_index_storage_format
= V2.");
+ }
+
for (String col : indexDef.getColumnNames()) {
Column column = olapTable.getColumn(col);
if (column != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 35d552e50f0..24bd1fdbfac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -1228,9 +1228,7 @@ public class PropertyAnalyzer {
invertedIndexFileStorageFormat =
properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
} else {
- if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
- return TInvertedIndexFileStorageFormat.V1;
- } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
+ if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
return TInvertedIndexFileStorageFormat.V2;
} else if
(Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) {
return TInvertedIndexFileStorageFormat.SNII;
@@ -1240,7 +1238,9 @@ public class PropertyAnalyzer {
}
if (invertedIndexFileStorageFormat.equalsIgnoreCase("v1")) {
- return TInvertedIndexFileStorageFormat.V1;
+ throw new AnalysisException(
+ "Inverted index V1 is deprecated and no longer allowed for
new index creation."
+ + " Please use inverted index V2.");
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v2")) {
return TInvertedIndexFileStorageFormat.V2;
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v3")) {
@@ -1248,9 +1248,7 @@ public class PropertyAnalyzer {
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("snii")) {
return TInvertedIndexFileStorageFormat.SNII;
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default"))
{
- if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
- return TInvertedIndexFileStorageFormat.V1;
- } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
+ if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
return TInvertedIndexFileStorageFormat.V2;
} else if
(Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) {
return TInvertedIndexFileStorageFormat.SNII;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 76dc96d889b..0764f06f6c0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -339,14 +339,17 @@ public class PropertyAnalyzerTest {
TInvertedIndexFileStorageFormat result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null);
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3,
result);
+ // Config=V1 with no explicit property: V1 config is ignored,
falls through to V3
Config.inverted_index_storage_format = "V1";
result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new HashMap<>());
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1,
result);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3,
result);
Map<String, String> propertiesWithV1 = new HashMap<>();
propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v1");
- result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1,
result);
+ AnalysisException v1Ex =
Assertions.assertThrows(AnalysisException.class,
+ () ->
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1));
+ Assertions.assertTrue(v1Ex.getMessage().contains(
+ "Inverted index V1 is deprecated and no longer allowed for
new index creation."));
Map<String, String> propertiesWithV2 = new HashMap<>();
propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"v2");
@@ -358,11 +361,12 @@ public class PropertyAnalyzerTest {
result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3);
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3,
result);
+ // "default" + Config=V1: falls through to V3 (current default
when config is not V2)
Config.inverted_index_storage_format = "V1";
Map<String, String> propertiesWithDefaultV1 = new HashMap<>();
propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
"default");
result =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1);
- Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1,
result);
+ Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3,
result);
Config.inverted_index_storage_format = "V2";
Map<String, String> propertiesWithDefaultV2 = new HashMap<>();
diff --git a/regression-test/data/fault_injection_p0/test_index_io_context.out
b/regression-test/data/fault_injection_p0/test_index_io_context.out
index b24411f7634..38dc7dc576b 100644
--- a/regression-test/data/fault_injection_p0/test_index_io_context.out
+++ b/regression-test/data/fault_injection_p0/test_index_io_context.out
@@ -8,51 +8,6 @@
-- !sql --
177
--- !sql --
-177
-
--- !sql --
-177
-
--- !sql --
-177
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
--- !sql --
-2
-
-- !sql --
2
@@ -98,18 +53,6 @@
-- !sql --
0
--- !sql --
-0
-
--- !sql --
-0
-
--- !sql --
-0
-
--- !sql --
-4
-
-- !sql --
4
@@ -119,42 +62,6 @@
-- !sql --
4
--- !sql --
-4
-
--- !sql --
-4
-
--- !sql --
-844
-
--- !sql --
-152
-
--- !sql --
-99
-
--- !sql --
-13
-
--- !sql --
-152
-
--- !sql --
-844
-
--- !sql --
-152
-
--- !sql --
-99
-
--- !sql --
-13
-
--- !sql --
-152
-
-- !sql --
844
diff --git a/regression-test/data/inverted_index_p0/test_index_multi_match.out
b/regression-test/data/inverted_index_p0/test_index_multi_match.out
index 86762cae36f..865303c7942 100644
--- a/regression-test/data/inverted_index_p0/test_index_multi_match.out
+++ b/regression-test/data/inverted_index_p0/test_index_multi_match.out
@@ -23,48 +23,12 @@
-- !sql --
44
--- !sql --
-178
-
--- !sql --
-180
-
--- !sql --
-859
-
--- !sql --
-44
-
--- !sql --
-178
-
--- !sql --
-180
-
--- !sql --
-859
-
--- !sql --
-44
-
-- !sql --
30
-- !sql --
30
--- !sql --
-30
-
--- !sql --
-30
-
--- !sql --
-5
-
--- !sql --
-5
-
-- !sql --
5
@@ -77,18 +41,6 @@
-- !sql --
30
--- !sql --
-30
-
--- !sql --
-30
-
--- !sql --
-5
-
--- !sql --
-5
-
-- !sql --
5
@@ -101,18 +53,6 @@
-- !sql --
30
--- !sql --
-30
-
--- !sql --
-30
-
--- !sql --
-5
-
--- !sql --
-5
-
-- !sql --
5
@@ -125,18 +65,6 @@
-- !sql --
30
--- !sql --
-30
-
--- !sql --
-30
-
--- !sql --
-5
-
--- !sql --
-5
-
-- !sql --
5
@@ -149,18 +77,6 @@
-- !sql --
30
--- !sql --
-30
-
--- !sql --
-30
-
--- !sql --
-5
-
--- !sql --
-5
-
-- !sql --
5
@@ -173,18 +89,6 @@
-- !sql --
30
--- !sql --
-30
-
--- !sql --
-30
-
--- !sql --
-5
-
--- !sql --
-5
-
-- !sql --
5
diff --git a/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
b/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
index 2f60a7c0836..553a57d61e1 100644
--- a/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
+++ b/regression-test/suites/ann_index_p0/create_ann_index_test.groovy
@@ -407,7 +407,7 @@ suite("create_ann_index_test") {
);
"""
- exception "ANN index is not supported in index format V1"
+ exception "Inverted index V1 is deprecated and no longer allowed for
new index creation."
}
// CREATE INDEX with ivf_on_disk type on existing table
diff --git
a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
index d52e8dd2acc..1c9aa22a040 100644
--- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
+++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
@@ -84,7 +84,7 @@ suite("test_mv_case") {
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
@@ -102,7 +102,7 @@ suite("test_mv_case") {
"colocate_with" = "dwd_info_group",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"group_commit_interval_ms" = "10000",
@@ -124,7 +124,7 @@ suite("test_mv_case") {
"colocate_with" = "dwd_info_group",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"group_commit_interval_ms" = "10000",
diff --git
a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
index a199ccaf6ec..8d073100691 100644
---
a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
@@ -160,7 +160,7 @@ suite("test_special_buffer_before_nested", "p0") {
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"store_row_column" = "true",
diff --git
a/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
index 0597d2a61d6..43e2dca9dc8 100644
---
a/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
@@ -35,7 +35,7 @@ suite("test_index_bkd_writer_fault_injection",
"nonConcurrent") {
DUPLICATE KEY(`id`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
+ PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V2");
"""
try {
diff --git
a/regression-test/suites/fault_injection_p0/test_index_builder_drop_index_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_builder_drop_index_fault_injection.groovy
index 353afdf0769..60be3e957ab 100644
---
a/regression-test/suites/fault_injection_p0/test_index_builder_drop_index_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_index_builder_drop_index_fault_injection.groovy
@@ -123,7 +123,6 @@ suite("test_index_builder_drop_index_fault_injection",
"nonConcurrent") {
runTest(indexTbName)
}
- createTestTable("v1")
createTestTable("v2")
}
\ No newline at end of file
diff --git
a/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
index b99499b4355..87e3e56c7a4 100644
---
a/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
@@ -285,7 +285,7 @@ suite("test_index_compaction_exception_fault_injection",
"nonConcurrent") {
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"disable_auto_compaction" = "true"
);
"""
diff --git
a/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy
index b33ce977bbe..daa452c8e6e 100644
---
a/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy
@@ -311,7 +311,7 @@ suite("test_index_compaction_failure_injection",
"nonConcurrent") {
DUPLICATE KEY(`id`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
+ PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V2");
"""
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
@@ -342,7 +342,7 @@ suite("test_index_compaction_failure_injection",
"nonConcurrent") {
"replication_num" = "1",
"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
);
"""
diff --git
a/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
b/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
index 4c5c00dd7e4..6e60b80816e 100644
--- a/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
+++ b/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
@@ -69,82 +69,49 @@ suite("test_index_io_context", "nonConcurrent") {
}
try {
- create_table(tableName1, "V1");
create_table(tableName2, "V2");
- load_httplogs_data.call(tableName1, 'test_index_io_context1', 'true',
'json', 'documents-1000.json')
load_httplogs_data.call(tableName2, 'test_index_io_context2', 'true',
'json', 'documents-1000.json')
sql "sync"
- sql """ set enable_segment_limit_pushdown = true; """
+ sql """ set enable_common_expr_pushdown = true; """
try {
GetDebugPoint().enableDebugPointForAllBEs("InvertedIndexReader.handle_searcher_cache.io_ctx")
- qt_sql """ select count() from ${tableName1} where request
match_any 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_any 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_any 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_any 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_any 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_any 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_all 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_all 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_all 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_all 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_all 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_all 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg ~10+'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg ~10+'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg ~10+'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg ~10+'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg ~10+'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg ~10+'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg ~10'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg ~10'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase 'ticket_quest_bg2.jpg ~10'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg ~10'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg ~10'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase 'ticket_quest_bg2.jpg ~10'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase_prefix 'ticket_quest_bg ~10'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase_prefix 'ticket_quest_bg ~10'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase_prefix 'ticket_quest_bg ~10'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase_prefix 'ticket_quest_bg ~10'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase_prefix 'ticket_quest_bg ~10'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase_prefix 'ticket_quest_bg ~10'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase_prefix 'quest'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase_prefix 'quest'; """
- qt_sql """ select count() from ${tableName1} where request
match_phrase_prefix 'quest'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase_prefix 'quest'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase_prefix 'quest'; """
qt_sql """ select count() from ${tableName2} where request
match_phrase_prefix 'quest'; """
- qt_sql """ select count() from ${tableName1} where status = '200';
"""
- qt_sql """ select count() from ${tableName1} where status = '304';
"""
- qt_sql """ select count() from ${tableName1} where size = '24736';
"""
- qt_sql """ select count() from ${tableName1} where size = '985';
"""
- qt_sql """ select count() from ${tableName1} where size = '0'; """
qt_sql """ select count() from ${tableName2} where status = '200';
"""
qt_sql """ select count() from ${tableName2} where status = '304';
"""
qt_sql """ select count() from ${tableName2} where size = '24736';
"""
qt_sql """ select count() from ${tableName2} where size = '985';
"""
qt_sql """ select count() from ${tableName2} where size = '0'; """
- qt_sql """ select count() from ${tableName1} where status = '200';
"""
- qt_sql """ select count() from ${tableName1} where status = '304';
"""
- qt_sql """ select count() from ${tableName1} where size = '24736';
"""
- qt_sql """ select count() from ${tableName1} where size = '985';
"""
- qt_sql """ select count() from ${tableName1} where size = '0'; """
qt_sql """ select count() from ${tableName2} where status = '200';
"""
qt_sql """ select count() from ${tableName2} where status = '304';
"""
qt_sql """ select count() from ${tableName2} where size = '24736';
"""
diff --git
a/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy
index f53d1c38443..6de80924425 100644
---
a/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy
@@ -19,32 +19,11 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_skip_index_compaction_fault_injection", "nonConcurrent") {
def isCloudMode = isCloudMode()
- def tableName1 = "test_skip_index_compaction_fault_injection_1"
def tableName2 = "test_skip_index_compaction_fault_injection_2"
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
- sql "DROP TABLE IF EXISTS ${tableName1}"
- sql """
- CREATE TABLE ${tableName1} (
- `@timestamp` int(11) NULL COMMENT "",
- `clientip` varchar(20) NULL COMMENT "",
- `request` text NULL COMMENT "",
- `status` int(11) NULL COMMENT "",
- `size` int(11) NULL COMMENT "",
- INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
- INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" =
"english", "support_phrase" = "true") COMMENT ''
- ) ENGINE=OLAP
- DUPLICATE KEY(`@timestamp`)
- COMMENT "OLAP"
- DISTRIBUTED BY RANDOM BUCKETS 1
- PROPERTIES (
- "replication_allocation" = "tag.location.default: 1",
- "disable_auto_compaction" = "true",
- "inverted_index_storage_format" = "V1"
- );
- """
sql "DROP TABLE IF EXISTS ${tableName2}"
sql """
@@ -177,12 +156,6 @@ suite("test_skip_index_compaction_fault_injection",
"nonConcurrent") {
has_update_be_config = true
check_config.call("inverted_index_compaction_enable", "true");
- try {
-
GetDebugPoint().enableDebugPointForAllBEs("Compaction::open_inverted_index_file_reader")
- run_test.call(tableName1)
- } finally {
-
GetDebugPoint().disableDebugPointForAllBEs("Compaction::open_inverted_index_file_reader")
- }
try {
GetDebugPoint().enableDebugPointForAllBEs("Compaction::open_inverted_index_file_writer")
diff --git
a/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy
index 8d1226824d5..d42ed687103 100644
---
a/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy
@@ -229,7 +229,7 @@
suite("test_write_inverted_index_exception_fault_injection", "nonConcurrent") {
"InvertedIndexColumnWriter::create_unsupported_type_for_inverted_index"
]
- def inverted_index_storage_format = ["v1", "v2"]
+ def inverted_index_storage_format = ["v2"]
try {
String backend_id;
backend_id = backendId_to_backendIP.keySet()[0]
diff --git
a/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
b/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
index 14e74d9b8eb..c019d37036b 100644
---
a/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
@@ -235,7 +235,7 @@ suite("test_build_index_exception_fault_injection",
"nonConcurrent,p2") {
"IndexBuilder::gc_output_rowset_is_local_rowset"
]
- def inverted_index_storage_format = ["v1", "v2"]
+ def inverted_index_storage_format = ["v2"]
inverted_index_storage_format.each { format ->
def tableName = "${tableNamePrefix}_${format}"
diff --git a/regression-test/suites/function_p0/test_array_map.groovy
b/regression-test/suites/function_p0/test_array_map.groovy
index 95fbb8e08f2..ef9ccecc3fe 100644
--- a/regression-test/suites/function_p0/test_array_map.groovy
+++ b/regression-test/suites/function_p0/test_array_map.groovy
@@ -111,7 +111,7 @@ suite("test_array_map") {
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
diff --git
a/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy
b/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy
index a7571934f73..e6497b92447 100644
---
a/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy
+++
b/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy
@@ -75,7 +75,7 @@ PROPERTIES (
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
-"inverted_index_storage_format" = "V1",
+"inverted_index_storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"group_commit_interval_ms" = "10000",
@@ -142,7 +142,7 @@ PROPERTIES (
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
-"inverted_index_storage_format" = "V1",
+"inverted_index_storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"group_commit_interval_ms" = "10000",
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy
index f602d114503..bc649228e54 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy
@@ -20,7 +20,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_index_change_1") {
sql "set enable_add_index_for_new_data = true"
def timeout = 60000
-
+
def tableName = "test_index_change_1"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -39,27 +39,27 @@ suite("test_index_change_1") {
sql """ INSERT INTO ${tableName} VALUES
(1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
"""
-
+
qt_select1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
// create inverted index
@@ -83,65 +83,4 @@ suite("test_index_change_1") {
qt_select5 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
qt_select6 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
- tableName = "test_index_change_1_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `note` TEXT COMMENT "备注")
- DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
- """
-
- qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
-
- // create inverted index
- sql """ CREATE INDEX idx_user_id ON ${tableName}(`user_id`) USING INVERTED
"""
-
- sql """ CREATE INDEX idx_note ON ${tableName}(`note`) using inverted
properties("support_phrase" = "true", "parser" = "english", "lower_case" =
"true") """
- wait_for_last_col_change_finish(tableName, timeout)
- sql """ CREATE INDEX idx_city ON ${tableName}(`city`) using inverted
properties("support_phrase" = "true", "parser" = "english", "lower_case" =
"true") """
- wait_for_last_col_change_finish(tableName, timeout)
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 3)
- assertEquals(show_result[0][2], "idx_user_id")
- assertEquals(show_result[1][2], "idx_note")
- assertEquals(show_result[2][2], "idx_city")
-
- qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy
index 2d3b2d7916c..91c535ab679 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy
@@ -46,7 +46,7 @@ suite("test_index_change_2") {
}
assertTrue(useTime <= OpTimeout,
"wait_for_latest_build_index_on_partition_finish timeout")
}
-
+
def tableName = "test_index_change_2"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -68,27 +68,27 @@ suite("test_index_change_2") {
sql """ INSERT INTO ${tableName} VALUES
(1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
"""
-
+
qt_select1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
qt_select2 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
qt_select3 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and
sex = 1 ORDER BY user_id; """
@@ -117,74 +117,4 @@ suite("test_index_change_2") {
qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- tableName = "test_index_change_2_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `note` TEXT COMMENT "备注",
- INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '',
- INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
- """
-
- qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
- qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- // create inverted index idx_city
- sql """ CREATE INDEX idx_city ON ${tableName}(`city`) using inverted
properties("support_phrase" = "true", "parser" = "english", "lower_case" =
"true") """
- wait_for_last_col_change_finish(tableName, timeout)
-
- // drop inverted index idx_user_id, idx_note
- sql """ DROP INDEX idx_user_id ON ${tableName} """
- wait_for_build_index_on_partition_finish(tableName, timeout)
- sql """ DROP INDEX idx_note ON ${tableName} """
- wait_for_build_index_on_partition_finish(tableName, timeout)
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 1)
- assertEquals(show_result[0][2], "idx_city")
-
- qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy
index f6048fc387c..858dac305ef 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy
@@ -21,7 +21,7 @@ suite("test_index_change_3") {
def timeout = 60000
sql "set enable_add_index_for_new_data = true"
-
+
def tableName = "test_index_change_3"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -43,27 +43,27 @@ suite("test_index_change_3") {
sql """ INSERT INTO ${tableName} VALUES
(1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
"""
-
+
qt_select1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
qt_select2 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
qt_select3 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and
sex = 1 ORDER BY user_id; """
@@ -95,79 +95,4 @@ suite("test_index_change_3") {
qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- tableName = "test_index_change_3_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `note` TEXT COMMENT "备注",
- INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '',
- INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
- """
-
- qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
- qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- // create inverted index idx_city
- sql """ CREATE INDEX idx_city ON ${tableName}(`city`) using inverted
properties("support_phrase" = "true", "parser" = "english", "lower_case" =
"true") """
- wait_for_last_col_change_finish(tableName, timeout)
-
- if (!isCloudMode()) {
- build_index_on_table("idx_city", tableName)
- wait_for_last_build_index_finish(tableName, timeout)
- }
-
- // drop inverted index idx_user_id, idx_note
- sql """ DROP INDEX idx_city ON ${tableName} """
- wait_for_last_build_index_finish(tableName, timeout)
- sql """ DROP INDEX idx_note ON ${tableName} """
- wait_for_last_build_index_finish(tableName, timeout)
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 1)
- assertEquals(show_result[0][2], "idx_user_id")
-
- qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy
index aac5c2233da..11314e5beae 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy
@@ -46,7 +46,7 @@ suite("test_index_change_4") {
}
assertTrue(useTime <= OpTimeout,
"wait_for_latest_build_index_on_partition_finish timeout")
}
-
+
def tableName = "test_index_change_4"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -68,27 +68,27 @@ suite("test_index_change_4") {
sql """ INSERT INTO ${tableName} VALUES
(1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
"""
-
+
qt_select1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
qt_select2 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
qt_select3 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and
sex = 1 ORDER BY user_id; """
@@ -121,78 +121,4 @@ suite("test_index_change_4") {
qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- tableName = "test_index_change_4_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `note` TEXT COMMENT "备注",
- INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '',
- INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
- """
-
- qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
- qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- // drop inverted index idx_user_id, idx_note
- sql """ DROP INDEX idx_user_id ON ${tableName} """
- wait_for_last_build_index_finish(tableName, timeout)
- sql """ DROP INDEX idx_note ON ${tableName} """
- wait_for_last_build_index_finish(tableName, timeout)
- // create inverted index idx_city
- sql """ CREATE INDEX idx_note ON ${tableName}(`note`) using inverted
properties("support_phrase" = "true", "parser" = "english", "lower_case" =
"true") """
- wait_for_last_col_change_finish(tableName, timeout)
- // build index
- if (!isCloudMode()) {
- build_index_on_table("idx_note", tableName)
- wait_for_build_index_on_partition_finish(tableName, timeout)
- }
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 1)
- assertEquals(show_result[0][2], "idx_note")
-
- qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy
index ea236eba0e1..59130f1adfe 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy
@@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_index_change_5") {
def timeout = 60000
-
+
def tableName = "test_index_change_5"
sql "set enable_add_index_for_new_data = true"
@@ -43,27 +43,27 @@ suite("test_index_change_5") {
sql """ INSERT INTO ${tableName} VALUES
(1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
"""
-
+
qt_select1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
qt_select2 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
qt_select3 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and
sex = 1 ORDER BY user_id; """
@@ -86,69 +86,4 @@ suite("test_index_change_5") {
qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- tableName = "test_index_change_5_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `note` TEXT COMMENT "备注",
- INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '',
- INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
- """
-
- qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
- qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- // drop inverted index idx_user_id, idx_note
- sql """ DROP INDEX idx_user_id ON ${tableName} """
- wait_for_last_build_index_finish(tableName, timeout)
- sql """ DROP INDEX idx_note ON ${tableName} """
- wait_for_last_build_index_finish(tableName, timeout)
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 0)
-
- qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy
index 76b5c321bda..5d8a0c9ab87 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy
@@ -17,7 +17,7 @@
import org.codehaus.groovy.runtime.IOGroovyMethods
-suite("test_index_change_6") {
+suite("test_index_change_6", "nonConcurrent") {
def timeout = 300000
def delta_time = 1000
@@ -48,7 +48,7 @@ suite("test_index_change_6") {
}
assertTrue(finished, "wait_for_build_index_on_partition_finish
timeout, latest result: ${alter_res}")
}
-
+
def tableName = "test_index_change_6"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -70,27 +70,27 @@ suite("test_index_change_6") {
sql """ INSERT INTO ${tableName} VALUES
(1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
"""
-
+
sql """ INSERT INTO ${tableName} VALUES
(6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
"""
-
+
qt_select1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
qt_select2 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
qt_select3 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and
sex = 1 ORDER BY user_id; """
@@ -131,87 +131,4 @@ suite("test_index_change_6") {
qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- tableName = "test_index_change_6_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `note` TEXT COMMENT "备注",
- INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '',
- INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a
scientist')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher')
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer')
- """
-
- qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex; """
- qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
-
- // drop inverted index idx_user_id, idx_note
- sql """ DROP INDEX idx_user_id ON ${tableName} """
- wait_for_build_index_on_partition_finish(tableName, timeout)
- sql """ DROP INDEX idx_note ON ${tableName} """
- wait_for_build_index_on_partition_finish(tableName, timeout)
-
- // create inverted index
- sql """ CREATE INDEX idx_user_id ON ${tableName}(`user_id`) USING INVERTED
"""
- build_index_on_table("idx_user_id", tableName)
- wait_for_build_index_on_partition_finish(tableName, timeout)
-
- sql """ CREATE INDEX idx_note ON ${tableName}(`note`) USING INVERTED
PROPERTIES("parser" = "english") """
- if (isCloudMode()) {
- wait_for_last_col_change_finish(tableName, timeout)
- }
-
- // build inverted index
- if (!isCloudMode()) {
- build_index_on_table("idx_note", tableName)
- wait_for_build_index_on_partition_finish(tableName, timeout)
- }
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 2)
- assertEquals(show_result[0][2], "idx_user_id")
- assertEquals(show_result[1][2], "idx_note")
-
- qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
ORDER BY user_id; """
- qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing'
and sex = 1 ORDER BY user_id; """
- qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer'
and sex = 0 ORDER BY user_id; """
- qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE
'electrical engineer' ORDER BY user_id; """
- qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer
Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
index 572b50c3c2e..4d4c2a7bc4e 100644
---
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
@@ -45,7 +45,7 @@ suite("test_index_change_on_new_column") {
}
assertTrue(useTime <= OpTimeout,
"wait_for_latest_build_index_on_partition_finish timeout")
}
-
+
def tableName = "test_index_change_on_new_column"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -61,10 +61,10 @@ suite("test_index_change_on_new_column") {
sql """ INSERT INTO ${tableName} VALUES
(1, 'hello world')
"""
-
+
// add new column
sql """ alter table ${tableName} add column s1 varchar(50) default null
after s; """
-
+
qt_select1 """ SELECT * FROM ${tableName}; """
// create inverted index on new column
@@ -131,48 +131,4 @@ suite("test_index_change_on_new_column") {
qt_select2 """ SELECT * FROM ${tableName} order by id; """
qt_select3 """ SELECT
/*+SET_VAR(enable_fallback_on_missing_inverted_index=false) */ * FROM
${tableName} where s1 match 'welcome'; """
qt_select4 """ SELECT
/*+SET_VAR(enable_fallback_on_missing_inverted_index=false) */ * FROM
${tableName} where s match 'hello world'; """
-
- tableName = "test_index_change_on_new_column1_index_v1"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- `id` INT COMMENT "",
- `s` STRING COMMENT "",
- INDEX idx_s(s) USING INVERTED
- )
- DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`)
- PROPERTIES ( "inverted_index_storage_format" = "v1", "replication_num"
= "1" );
- """
-
- sql """ INSERT INTO ${tableName} VALUES
- (1, 'hello world')
- """
-
- // add new column
- sql """ alter table ${tableName} add column s1 varchar(50) default null
after s; """
-
- qt_select1 """ SELECT * FROM ${tableName}; """
-
- // create inverted index on new column
- sql """ alter table ${tableName} add index idx_s1(s1) USING INVERTED
PROPERTIES('parser' = 'english')"""
- wait_for_last_col_change_finish(tableName, timeout)
-
- sql """ INSERT INTO ${tableName} VALUES
- (2, 'hello wold', 'welcome to the world')
- """
- // build inverted index on new column
- if (!isCloudMode()) {
- build_index_on_table("idx_s1", tableName)
- wait_for_build_index_on_partition_finish(tableName, timeout)
- }
-
- show_result = sql "show index from ${tableName}"
- logger.info("show index from " + tableName + " result: " + show_result)
- assertEquals(show_result.size(), 2)
- assertEquals(show_result[0][2], "idx_s")
- assertEquals(show_result[1][2], "idx_s1")
- qt_select2 """ SELECT * FROM ${tableName} order by id; """
- qt_select3 """ SELECT
/*+SET_VAR(enable_fallback_on_missing_inverted_index=false) */ * FROM
${tableName} where s1 match 'welcome'; """
- qt_select4 """ SELECT
/*+SET_VAR(enable_fallback_on_missing_inverted_index=false) */ * FROM
${tableName} where s match 'hello world'; """
}
diff --git
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
index b4773eab32f..eddec60c196 100644
---
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
+++
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
@@ -105,7 +105,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") {
DUPLICATE KEY(`id`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
+ PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V2");
"""
sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple",
100); """
diff --git
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy
index 97e86e46439..266c9641979 100644
---
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy
+++
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy
@@ -271,7 +271,7 @@ suite("test_index_compaction_null", "nonConcurrent") {
"disable_auto_compaction" = "true",
"in_memory" = "false",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
)
"""
@@ -307,7 +307,7 @@ suite("test_index_compaction_null", "nonConcurrent") {
"enable_unique_key_merge_on_write" = "true",
"in_memory" = "false",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
)
"""
sql """ set enable_segment_limit_pushdown = true """
diff --git
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
index cd9357d098f..d4476af58fc 100644
---
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
+++
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
@@ -109,7 +109,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent")
{
"replication_num" = "1",
"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
);
"""
sql """ set enable_segment_limit_pushdown = true """
diff --git
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
index 43a9ad99e37..3aa089840eb 100644
---
a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
+++
b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
@@ -120,7 +120,7 @@ suite("test_index_compaction_with_multi_index_segments",
"nonConcurrent") {
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"disable_auto_compaction" = "true",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
);
"""
@@ -248,7 +248,7 @@ suite("test_index_compaction_with_multi_index_segments",
"nonConcurrent") {
"replication_allocation" = "tag.location.default: 1",
"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
);
"""
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_index_change_format.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_index_change_format.groovy
index 8a5179b0010..0ca7a401136 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_index_change_format.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_index_change_format.groovy
@@ -46,6 +46,5 @@ suite("test_index_change_format", "inverted_index_format_v2"){
}
}
- createAndAlterTable("test_index_change_format_v1", "V1", "V2")
createAndAlterTable("test_index_change_format_v2", "V2", "V1")
}
diff --git a/regression-test/suites/inverted_index_p0/load/test_insert.groovy
b/regression-test/suites/inverted_index_p0/load/test_insert.groovy
index b2dbfdf1f5c..43c2c197b9f 100644
--- a/regression-test/suites/inverted_index_p0/load/test_insert.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_insert.groovy
@@ -76,7 +76,6 @@ suite("test_insert_with_index", "p0, nonConcurrent") {
if (isCloudMode()) {
return;
}
- test.call("V1")
}
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "false")
@@ -84,8 +83,7 @@ suite("test_insert_with_index", "p0, nonConcurrent") {
if (isCloudMode()) {
return;
}
- test.call("V1")
}
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "true")
-}
\ No newline at end of file
+}
diff --git
a/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
b/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
index bcd2bd34b80..a2b1e5c6583 100644
--- a/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
@@ -168,10 +168,8 @@ suite("test_spark_load_with_index_p0", "p0") {
}
set_be_config("inverted_index_ram_dir_enable", "true")
- test.call("V1")
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "false")
- test.call("V1")
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "true")
}
diff --git
a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
index 73a68455ced..7b71e49c970 100644
--- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
@@ -109,7 +109,6 @@ suite("test_stream_load_with_inverted_index_p0", "p0,
nonConcurrent") {
if (isCloudMode()) {
return;
}
- test.call("V1")
}
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "false")
diff --git
a/regression-test/suites/inverted_index_p0/storage_format/test_inverted_index_v1_deprecated.groovy
b/regression-test/suites/inverted_index_p0/storage_format/test_inverted_index_v1_deprecated.groovy
new file mode 100644
index 00000000000..5879d82b9f3
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/storage_format/test_inverted_index_v1_deprecated.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_inverted_index_v1_deprecated", "p0") {
+ sql "set enable_add_index_for_new_data = true"
+
+ // explicit v1 (lowercase) must be rejected
+ test {
+ sql """
+ CREATE TABLE test_v1_rejected (
+ k INT,
+ v STRING,
+ INDEX idx_v (v) USING INVERTED
+ ) ENGINE=OLAP DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "inverted_index_storage_format" = "v1"
+ )
+ """
+ exception "Inverted index V1 is deprecated and no longer allowed for
new index creation."
+ }
+
+ // uppercase V1 must also be rejected
+ test {
+ sql """
+ CREATE TABLE test_v1_rejected_upper (
+ k INT,
+ v STRING,
+ INDEX idx_v (v) USING INVERTED
+ ) ENGINE=OLAP DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "inverted_index_storage_format" = "V1"
+ )
+ """
+ exception "Inverted index V1 is deprecated and no longer allowed for
new index creation."
+ }
+
+ // no format specified -> default succeeds and format is not V1
+ // ALTER TABLE ADD INDEX on default table succeeds
+ try {
+ sql "DROP TABLE IF EXISTS test_v1_deprecated_default"
+ sql """
+ CREATE TABLE test_v1_deprecated_default (
+ k INT,
+ v STRING
+ ) ENGINE=OLAP DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ )
+ """
+ def showCreate = sql "SHOW CREATE TABLE test_v1_deprecated_default"
+ assertTrue(showCreate.size() > 0)
+
assertFalse(showCreate[0][1].contains("\"inverted_index_storage_format\" =
\"V1\""))
+
+ sql "ALTER TABLE test_v1_deprecated_default ADD INDEX idx_v (v) USING
INVERTED"
+ def showCreateAfter = sql "SHOW CREATE TABLE
test_v1_deprecated_default"
+ assertTrue(showCreateAfter[0][1].contains("idx_v"))
+
assertFalse(showCreateAfter[0][1].contains("\"inverted_index_storage_format\" =
\"V1\""))
+ } finally {
+ sql "DROP TABLE IF EXISTS test_v1_deprecated_default"
+ }
+}
diff --git
a/regression-test/suites/inverted_index_p0/test_compound_reader_fault_injection.groovy
b/regression-test/suites/inverted_index_p0/test_compound_reader_fault_injection.groovy
index 8cf49742fe3..cf2c18d8b02 100644
---
a/regression-test/suites/inverted_index_p0/test_compound_reader_fault_injection.groovy
+++
b/regression-test/suites/inverted_index_p0/test_compound_reader_fault_injection.groovy
@@ -35,7 +35,7 @@ suite("test_compound_reader_fault_injection",
"nonConcurrent") {
DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
- "inverted_index_storage_format" = "V1"
+ "inverted_index_storage_format" = "V2"
);
"""
diff --git
a/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy
b/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy
index de9a1a845ed..d69224ce206 100644
--- a/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy
@@ -34,12 +34,10 @@ suite("test_index_chinese_column", "inverted_index_select"){
qt_sql "SELECT * FROM ${table_name} WHERE 名称 match_all 'json'"
}
- def table_name_v1 = "test_index_chinese_column_v1"
def table_name_v2 = "test_index_chinese_column_v2"
sql "set enable_unicode_name_support=true"
sql """ set enable_segment_limit_pushdown = true """
- createAndInsertData(table_name_v1, "V1")
createAndInsertData(table_name_v2, "V2")
}
diff --git
a/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy
b/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy
index fdfbc5c57ff..5c68dd62319 100644
--- a/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy
@@ -17,14 +17,10 @@
suite("test_index_multi_match", "p0"){
- def indexTbName1 = "test_index_multi_match_1"
def indexTbName2 = "test_index_multi_match_2"
- def indexTbName3 = "test_index_multi_match_3"
def indexTbName4 = "test_index_multi_match_4"
- sql "DROP TABLE IF EXISTS ${indexTbName1}"
sql "DROP TABLE IF EXISTS ${indexTbName2}"
- sql "DROP TABLE IF EXISTS ${indexTbName3}"
sql "DROP TABLE IF EXISTS ${indexTbName4}"
def create_table = {table_name, idx_version ->
@@ -89,95 +85,57 @@ suite("test_index_multi_match", "p0"){
}
try {
- create_table(indexTbName1, 'V1')
create_table(indexTbName2, 'V2')
- create_table(indexTbName3, 'V1')
create_table(indexTbName4, 'V2')
- load_httplogs_data.call(indexTbName1, 'test_index_multi_match_1',
'true', 'json', 'documents-1000.json')
load_httplogs_data.call(indexTbName2, 'test_index_multi_match_2',
'true', 'json', 'documents-1000.json')
- load_httplogs_data.call(indexTbName3, 'test_index_multi_match_3',
'true', 'json', 'documents-1000.json')
load_httplogs_data.call(indexTbName4, 'test_index_multi_match_4',
'true', 'json', 'documents-1000.json')
sql "sync"
sql """ set enable_segment_limit_pushdown = true """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_phrase_prefix '2'); """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_phrase_prefix '2' or request match_phrase_prefix '2'); """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_phrase_prefix '2' or request match_phrase_prefix '2' or status
match_phrase_prefix '2' or size match_phrase_prefix '2'); """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_phrase_prefix 'a' or request match_phrase_prefix 'a' or status
match_phrase_prefix 'a' or size match_phrase_prefix 'a'); """
-
qt_sql """ select count() from ${indexTbName2} where (clientip
match_phrase_prefix '2'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_phrase_prefix '2' or request match_phrase_prefix '2'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_phrase_prefix '2' or request match_phrase_prefix '2' or status
match_phrase_prefix '2' or size match_phrase_prefix '2'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_phrase_prefix 'a' or request match_phrase_prefix 'a' or status
match_phrase_prefix 'a' or size match_phrase_prefix 'a'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, 'phrase_prefix', '2'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, 'phrase_prefix', '2'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'phrase_prefix', '2'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'phrase_prefix', 'a'); """
-
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, 'phrase_prefix', '2'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, 'phrase_prefix', '2'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'phrase_prefix', '2'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'phrase_prefix', 'a'); """
// match_any
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_any '2' or request match_any '2' or status match_any '2' or size
match_any '2'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_any '2' or request match_any '2' or status match_any '2' or size
match_any '2'); """
- qt_sql """ select count() from ${indexTbName3} where (clientip
match_any '2' or request match_any '2' or status match_any '2' or size
match_any '2'); """
qt_sql """ select count() from ${indexTbName4} where (clientip
match_any '2' or request match_any '2' or status match_any '2' or size
match_any '2'); """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_any 'a' or request match_any 'a' or status match_any 'a' or size
match_any 'a'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_any 'a' or request match_any 'a' or status match_any 'a' or size
match_any 'a'); """
- qt_sql """ select count() from ${indexTbName3} where (clientip
match_any 'a' or request match_any 'a' or status match_any 'a' or size
match_any 'a'); """
qt_sql """ select count() from ${indexTbName4} where (clientip
match_any 'a' or request match_any 'a' or status match_any 'a' or size
match_any 'a'); """
- qt_sql """ select count() from ${indexTbName1} where
multi_match(clientip, request, status, size, 'any', '2'); """
qt_sql """ select count() from ${indexTbName2} where
multi_match(clientip, request, status, size, 'any', '2'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'any', '2'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'any', '2'); """
- qt_sql """ select count() from ${indexTbName1} where
multi_match(clientip, request, status, size, 'any', 'a'); """
qt_sql """ select count() from ${indexTbName2} where
multi_match(clientip, request, status, size, 'any', 'a'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'any', 'a'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'any', 'a'); """
// match_all
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_all '2' or request match_all '2' or status match_all '2' or size
match_all '2'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_all '2' or request match_all '2' or status match_all '2' or size
match_all '2'); """
- qt_sql """ select count() from ${indexTbName3} where (clientip
match_all '2' or request match_all '2' or status match_all '2' or size
match_all '2'); """
qt_sql """ select count() from ${indexTbName4} where (clientip
match_all '2' or request match_all '2' or status match_all '2' or size
match_all '2'); """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_all 'a' or request match_all 'a' or status match_all 'a' or size
match_all 'a'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_all 'a' or request match_all 'a' or status match_all 'a' or size
match_all 'a'); """
- qt_sql """ select count() from ${indexTbName3} where (clientip
match_all 'a' or request match_all 'a' or status match_all 'a' or size
match_all 'a'); """
qt_sql """ select count() from ${indexTbName4} where (clientip
match_all 'a' or request match_all 'a' or status match_all 'a' or size
match_all 'a'); """
- qt_sql """ select count() from ${indexTbName1} where
multi_match(clientip, request, status, size, 'all', '2'); """
qt_sql """ select count() from ${indexTbName2} where
multi_match(clientip, request, status, size, 'all', '2'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'all', '2'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'all', '2'); """
- qt_sql """ select count() from ${indexTbName1} where
multi_match(clientip, request, status, size, 'all', 'a'); """
qt_sql """ select count() from ${indexTbName2} where
multi_match(clientip, request, status, size, 'all', 'a'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'all', 'a'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'all', 'a'); """
// match_phrase
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_phrase '2' or request match_phrase '2' or status match_phrase '2' or size
match_phrase '2'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_phrase '2' or request match_phrase '2' or status match_phrase '2' or size
match_phrase '2'); """
- qt_sql """ select count() from ${indexTbName3} where (clientip
match_phrase '2' or request match_phrase '2' or status match_phrase '2' or size
match_phrase '2'); """
qt_sql """ select count() from ${indexTbName4} where (clientip
match_phrase '2' or request match_phrase '2' or status match_phrase '2' or size
match_phrase '2'); """
- qt_sql """ select count() from ${indexTbName1} where (clientip
match_phrase 'a' or request match_phrase 'a' or status match_phrase 'a' or size
match_phrase 'a'); """
qt_sql """ select count() from ${indexTbName2} where (clientip
match_phrase 'a' or request match_phrase 'a' or status match_phrase 'a' or size
match_phrase 'a'); """
- qt_sql """ select count() from ${indexTbName3} where (clientip
match_phrase 'a' or request match_phrase 'a' or status match_phrase 'a' or size
match_phrase 'a'); """
qt_sql """ select count() from ${indexTbName4} where (clientip
match_phrase 'a' or request match_phrase 'a' or status match_phrase 'a' or size
match_phrase 'a'); """
- qt_sql """ select count() from ${indexTbName1} where
multi_match(clientip, request, status, size, 'phrase', '2'); """
qt_sql """ select count() from ${indexTbName2} where
multi_match(clientip, request, status, size, 'phrase', '2'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'phrase', '2'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'phrase', '2'); """
- qt_sql """ select count() from ${indexTbName1} where
multi_match(clientip, request, status, size, 'phrase', 'a'); """
qt_sql """ select count() from ${indexTbName2} where
multi_match(clientip, request, status, size, 'phrase', 'a'); """
- qt_sql """ select count() from ${indexTbName3} where
multi_match(clientip, request, status, size, 'phrase', 'a'); """
qt_sql """ select count() from ${indexTbName4} where
multi_match(clientip, request, status, size, 'phrase', 'a'); """
} finally {
diff --git
a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
index d95985176f7..85b3bf5c25f 100644
--- a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
@@ -128,7 +128,7 @@ suite("test_index_rqg_bug8", "test_index_rqg_bug8"){
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
diff --git
a/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
b/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
index b0fee7cc557..359ba30286c 100644
---
a/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
+++
b/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
@@ -96,10 +96,8 @@ suite("test_inverted_index_file_size", "nonConcurrent"){
try {
GetDebugPoint().enableDebugPointForAllBEs("file_size_not_in_rowset_meta")
set_be_config.call("inverted_index_compaction_enable", "true")
- test_table.call("V1")
test_table.call("V2")
set_be_config.call("inverted_index_compaction_enable", "false")
- test_table.call("V1")
test_table.call("V2")
} finally {
GetDebugPoint().disableDebugPointForAllBEs("file_size_not_in_rowset_meta")
diff --git
a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action.groovy
b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action.groovy
index beb11f3f40d..f1ec658da3e 100644
---
a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action.groovy
+++
b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action.groovy
@@ -93,6 +93,5 @@ suite("test_show_nested_index_file_http_action") {
}
}
- run_test("V1")
run_test("V2")
}
diff --git
a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
index 9caee9dc49b..c11c395fc61 100644
---
a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
+++
b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
@@ -129,7 +129,6 @@
suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0"
if (isCloudMode()) {
run_test("V2")
} else {
- run_test("V1")
run_test("V2")
}
diff --git
a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
deleted file mode 100644
index be0c76ab588..00000000000
---
a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_variant_index_format_v1", "p2, nonConcurrent") {
- if (isCloudMode()) {
- return;
- }
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
- def set_be_config = { key, value ->
- String backend_id;
- def backendId_to_backendIP = [:]
- def backendId_to_backendHttpPort = [:]
- getBackendIpHttpPort(backendId_to_backendIP,
backendId_to_backendHttpPort);
-
- backend_id = backendId_to_backendIP.keySet()[0]
- def (code, out, err) =
update_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), key, value)
- logger.info("update config: code=" + code + ", out=" + out + ", err="
+ err)
- }
-
- def load_json_data = {table_name, file_name ->
- // load the json data
- streamLoad {
- table "${table_name}"
-
- // set http request header params
- set 'read_json_by_line', 'true'
- set 'format', 'json'
- set 'max_filter_ratio', '0.1'
- file file_name // import json file
- time 10000 // limit inflight 10s
-
- // if declared a check callback, the default check condition will
ignore.
- // So you must check all condition
-
- check { result, exception, startTime, endTime ->
- if (exception != null) {
- throw exception
- }
- logger.info("Stream load ${file_name} result:
${result}".toString())
- def json = parseJson(result)
- assertEquals("success", json.Status.toLowerCase())
- // assertEquals(json.NumberTotalRows, json.NumberLoadedRows +
json.NumberUnselectedRows)
- assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
- }
- }
- }
-
- def table_name = "github_events"
- sql """DROP TABLE IF EXISTS ${table_name}"""
- setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
- sql """
- CREATE TABLE IF NOT EXISTS ${table_name} (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
- """
- }
- set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes",
"6294967296")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-1.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-2.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-3.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2022-11-07-16.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2022-11-07-10.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2022-11-07-22.json'}""")
- load_json_data.call(table_name, """${getS3Url() +
'/regression/gharchive.m/2022-11-07-23.json'}""")
- def backendId_to_backendIP = [:]
- def backendId_to_backendHttpPort = [:]
- getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
-
- def tablets = sql_return_maparray """ show tablets from ${table_name}; """
- String tablet_id = tablets[0].TabletId
- String backend_id = tablets[0].BackendId
- String ip = backendId_to_backendIP.get(backend_id)
- String port = backendId_to_backendHttpPort.get(backend_id)
- def (code_0, out_0, err_0) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code_0 + ", out=" +
out_0 + ", err=" + err_0)
- assertTrue(code_0 == 0)
- assertTrue(out_0.contains("crc_value"))
- assertTrue(out_0.contains("used_time_ms"))
- assertEquals("0", parseJson(out_0.trim()).start_version)
- assertEquals("9", parseJson(out_0.trim()).end_version)
- assertEquals("9", parseJson(out_0.trim()).rowset_count)
-
- qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int)
from github_events where cast(v["repo"]["name"] as string) =
'xpressengine/xe-core' order by 1;"""
- qt_sql """select count() from github_events where cast(v["repo"]["name"]
as string) = 'xpressengine/xe-core'"""
- set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes",
"2147483648")
-}
diff --git a/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
b/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
index a4a8d05e4a0..1c7740c602b 100644
--- a/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
+++ b/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
@@ -34,7 +34,7 @@ suite ("accept_null") {
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
diff --git
a/regression-test/suites/query_p0/show/test_nereids_show_build_index.groovy
b/regression-test/suites/query_p0/show/test_nereids_show_build_index.groovy
index 73e87efb103..b8bb50b29b2 100644
--- a/regression-test/suites/query_p0/show/test_nereids_show_build_index.groovy
+++ b/regression-test/suites/query_p0/show/test_nereids_show_build_index.groovy
@@ -33,7 +33,7 @@ suite("test_nereids_show_build_index") {
INDEX idx_note1 (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
)
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
+ PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V2" );
"""
sql """
CREATE TABLE IF NOT EXISTS
test_show_build_index.test_show_build_index_tbl2 (
@@ -47,7 +47,7 @@ suite("test_nereids_show_build_index") {
INDEX idx_note2 (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
)
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
+ PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V2" );
"""
sql "BUILD INDEX idx_user_id1 ON
test_show_build_index.test_show_build_index_tbl1;"
sql "BUILD INDEX idx_user_id2 ON
test_show_build_index.test_show_build_index_tbl2;"
diff --git
a/regression-test/suites/query_p0/show/test_nereids_show_index.groovy
b/regression-test/suites/query_p0/show/test_nereids_show_index.groovy
index e9ec03c1b91..0ab26bb6caf 100644
--- a/regression-test/suites/query_p0/show/test_nereids_show_index.groovy
+++ b/regression-test/suites/query_p0/show/test_nereids_show_index.groovy
@@ -32,7 +32,7 @@ suite("test_nereids_show_index") {
INDEX idx_note1 (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
)
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
+ PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V2" );
"""
sql """
CREATE TABLE IF NOT EXISTS test_show_index.test_show_index_tbl2 (
@@ -46,7 +46,7 @@ suite("test_nereids_show_index") {
INDEX idx_note2 (`note`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
)
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY
HASH(`user_id`)
- PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V1" );
+ PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format"
= "V2" );
"""
checkNereidsExecute("show index from
test_show_index.test_show_index_tbl2;")
diff --git
a/regression-test/suites/show_data_p2/test_table_property/test_cloud_inverted_index_v1_show_data.groovy
b/regression-test/suites/show_data_p2/test_table_property/test_cloud_inverted_index_v1_show_data.groovy
deleted file mode 100644
index 1077e0436a0..00000000000
---
a/regression-test/suites/show_data_p2/test_table_property/test_cloud_inverted_index_v1_show_data.groovy
+++ /dev/null
@@ -1,101 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// The cases is copied from https://github.com/trinodb/trino/tree/master
-// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/tpcds
-// and modified by Doris.
-import org.codehaus.groovy.runtime.IOGroovyMethods
-
- // loading one data 10 times, expect data size not rising
-suite("test_cloud_inverted_index_v1_show_data","p2, nonConcurrent") {
- //cloud-mode
- if (!isCloudMode()) {
- logger.info("not cloud mode, not run")
- return
- }
-
- def main = {
- def tableName="test_cloud_inverted_index_v1_show_data"
- sql "DROP TABLE IF EXISTS ${tableName};"
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName}(
- L_ORDERKEY INTEGER NOT NULL,
- L_PARTKEY INTEGER NOT NULL,
- L_SUPPKEY INTEGER NOT NULL,
- L_LINENUMBER INTEGER NOT NULL,
- L_QUANTITY DECIMAL(15,2) NOT NULL,
- L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
- L_DISCOUNT DECIMAL(15,2) NOT NULL,
- L_TAX DECIMAL(15,2) NOT NULL,
- L_RETURNFLAG CHAR(1) NOT NULL,
- L_LINESTATUS CHAR(1) NOT NULL,
- L_SHIPDATE DATE NOT NULL,
- L_COMMITDATE DATE NOT NULL,
- L_RECEIPTDATE DATE NOT NULL,
- L_SHIPINSTRUCT CHAR(25) NOT NULL,
- L_SHIPMODE CHAR(10) NOT NULL,
- L_COMMENT VARCHAR(44) NOT NULL,
- L_NULL VARCHAR,
- index index_SHIPINSTRUCT (L_SHIPINSTRUCT) using inverted,
- index index_SHIPMODE (L_SHIPMODE) using inverted,
- index index_COMMENT (L_COMMENT) using inverted
- )
- UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
- DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
- PROPERTIES (
- "inverted_index_storage_format" = "V1",
- "replication_num" = "1"
- )
- """
- List<String> tablets = get_tablets_from_table(tableName)
- def loadTimes = [1, 10]
- Map<String, List> sizeRecords = ["apiSize":[], "mysqlSize":[],
"cbsSize":[]]
- for (int i in loadTimes){
- // stream load 1 time, record each size
- repeate_stream_load_same_data(tableName, i,
"regression/tpch/sf0.1/lineitem.tbl.gz")
- def rows = sql_return_maparray "select count(*) as count from
${tableName};"
- logger.info("table ${tableName} has ${rows[0]["count"]} rows")
- // 加一下触发compaction的机制
- trigger_compaction(tablets)
-
- // 然后 sleep 1min， 等fe汇报完
- sleep(10 * 1000)
- sql "select count(*) from ${tableName}"
- sleep(10 * 1000)
-
-
sizeRecords["apiSize"].add(caculate_table_data_size_through_api(tablets))
-
sizeRecords["cbsSize"].add(caculate_table_data_size_in_backend_storage(tablets))
-
sizeRecords["mysqlSize"].add(show_table_data_size_through_mysql(tableName))
- logger.info("after ${i} times stream load, mysqlSize is:
${sizeRecords["mysqlSize"][-1]}, apiSize is: ${sizeRecords["apiSize"][-1]},
storageSize is: ${sizeRecords["cbsSize"][-1]}")
- }
-
- // expect mysqlSize == apiSize == storageSize
- assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["apiSize"][0])
- assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["cbsSize"][0])
- // expect load 1 times == load 10 times
- logger.info("after 1 time stream load, size is
${sizeRecords["mysqlSize"][0]}, after 10 times stream load, size is
${sizeRecords["mysqlSize"][1]}")
- assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["mysqlSize"][1])
- assertEquals(sizeRecords["apiSize"][0], sizeRecords["apiSize"][1])
- assertEquals(sizeRecords["cbsSize"][0], sizeRecords["cbsSize"][1])
- }
-
- set_config_before_show_data_test()
- sleep(10 * 1000)
- main()
- set_config_after_show_data_test()
- sleep(10 * 1000)
-}
diff --git
a/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy
b/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy
index e70b3507f0c..5f828f76d95 100644
---
a/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy
+++
b/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy
@@ -29,10 +29,7 @@ suite("array_contains_with_index_doc_value", "p0") {
sql "DROP TABLE IF EXISTS ${indexTblName}"
sql """ set default_variant_enable_doc_mode = true """
// create 1 replica table
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- if (storageFormat == "V1" && isCloudMode()) {
- return;
- }
+ def storageFormat = "V2"
sql """
CREATE TABLE IF NOT EXISTS `${indexTblName}` (
`apply_date` date NULL COMMENT '',
diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy
b/regression-test/suites/variant_p0/mv/multi_slot.groovy
index 68244ad82b8..a85b3d4e6e0 100644
--- a/regression-test/suites/variant_p0/mv/multi_slot.groovy
+++ b/regression-test/suites/variant_p0/mv/multi_slot.groovy
@@ -71,7 +71,7 @@ suite ("multi_slot") {
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
- "inverted_index_storage_format" = "V1",
+ "inverted_index_storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"replication_num" = "1"
diff --git
a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy
b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy
index 88079e08496..f832ccc716d 100644
--- a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy
+++ b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy
@@ -282,18 +282,6 @@ suite("test_predefine_ddl", "p0") {
exception("invalid INVERTED index")
}
- test {
- sql "DROP TABLE IF EXISTS ${tableName}"
- sql """CREATE TABLE ${tableName} (
- `id` bigint NULL,
- `var` string NULL,
- INDEX idx_ab (var) USING INVERTED PROPERTIES("parser"="unicode",
"support_phrase" = "true") COMMENT '',
- INDEX idx_ab_2 (var) USING INVERTED
- ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`)
- BUCKETS 1 PROPERTIES ( "replication_allocation" =
"tag.location.default: 1", "disable_auto_compaction" = "true",
"inverted_index_storage_format" = "v1")"""
- exception("column: var cannot have multiple inverted indexes with file
storage format: V1")
- }
-
test {
sql """CREATE TABLE ${tableName} (
`id` bigint NULL,
diff --git
a/regression-test/suites/variant_p0/v2/with_index/test_array_contains_with_inverted_index.groovy
b/regression-test/suites/variant_p0/v2/with_index/test_array_contains_with_inverted_index.groovy
index bcb1eed78a8..3f3bd1d133e 100644
---
a/regression-test/suites/variant_p0/v2/with_index/test_array_contains_with_inverted_index.groovy
+++
b/regression-test/suites/variant_p0/v2/with_index/test_array_contains_with_inverted_index.groovy
@@ -32,10 +32,7 @@ suite("test_array_contains_with_inverted_index",
"p0,nonConcurrent") {
sql "DROP TABLE IF EXISTS ${indexTblName}"
sql """ set default_variant_enable_doc_mode = false """
// create 1 replica table
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- if (storageFormat == "V1" && isCloudMode()) {
- return;
- }
+ def storageFormat = "V2"
sql """
CREATE TABLE IF NOT EXISTS `${indexTblName}` (
`apply_date` date NULL COMMENT '',
diff --git
a/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
b/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
index 5e10ee31550..c9333d52f7e 100644
---
a/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
+++
b/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
@@ -32,10 +32,7 @@ suite("test_array_contains_with_inverted_index",
"p0,nonConcurrent") {
sql "DROP TABLE IF EXISTS ${indexTblName}"
sql """ set default_variant_enable_doc_mode = false """
// create 1 replica table
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- if (storageFormat == "V1" && isCloudMode()) {
- return;
- }
+ def storageFormat = "V2"
sql """
CREATE TABLE IF NOT EXISTS `${indexTblName}` (
`apply_date` date NULL COMMENT '',
diff --git
a/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
b/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
index 1c351feaaeb..4044b9c7d06 100644
--- a/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
+++ b/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
@@ -28,10 +28,7 @@ suite("test_array_index_write", "nonConcurrent"){
stmt += strTmp
stmt = stmt.substring(0, stmt.length()-2)
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- if (isCloudMode()) {
- storageFormat = "V2";
- }
+ def storageFormat = "V2"
stmt += ") \nENGINE=OLAP\n" +
"DUPLICATE KEY(`k1`)\n" +
"COMMENT 'OLAP'\n" +
diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy
b/regression-test/suites/variant_p0/with_index/var_index.groovy
index 7edbc764da3..db31ff71ec7 100644
--- a/regression-test/suites/variant_p0/with_index/var_index.groovy
+++ b/regression-test/suites/variant_p0/with_index/var_index.groovy
@@ -53,111 +53,6 @@ suite("regression_test_variant_var_index", "p0,
nonConcurrent"){
trigger_and_wait_compaction(table_name, "full", 1800)
qt_sql "select * from var_index order by k limit 15"
- sql "DROP TABLE IF EXISTS var_index"
- boolean findException = false
- try {
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
- """
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("not supported in inverted index
format V1"))
- findException = true
- }
- assertTrue(findException)
-
- findException = false
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
- """
-
- try {
- sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("not supported in inverted index
format V1"))
- findException = true
- }
- assertTrue(findException)
-
- setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
- if (isCloudMode()) {
- sql "DROP TABLE IF EXISTS var_index"
- try {
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1",
"disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
- """
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("not supported in inverted
index format V1"))
- }
-
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction"
= "true", "inverted_index_storage_format" = "V1");
- """
-
- try {
- sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING
INVERTED"""
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("not supported in inverted
index format V1"))
- }
- } else {
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction"
= "true", "inverted_index_storage_format" = "V1");
- """
-
- sql "DROP TABLE IF EXISTS var_index"
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction"
= "true", "inverted_index_storage_format" = "V1");
- """
- sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
- test {
- sql """ build index idx_var on var_index"""
- exception "The idx_var index can not be built on the v column,
because it is a variant type column"
- }
- }
-
- }
sql """ DROP TABLE IF EXISTS ${table_name} """
sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
