This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new fb58b1380ee [test](inverted index) refactor stream load regression
test case (#38158) (#40778)
fb58b1380ee is described below
commit fb58b1380ee282cef21576eb29c7b407f995160e
Author: Sun Chenyang <[email protected]>
AuthorDate: Tue Sep 24 17:40:24 2024 +0800
[test](inverted index) refactor stream load regression test case (#38158)
(#40778)
## Proposed changes
pick from master #38158
<!--Describe your changes.-->
---
.../inverted_index_p0/load/test_stream_load.out | 36 +-----
.../inverted_index_p0/load/test_stream_load.groovy | 128 ++++++++-------------
2 files changed, 50 insertions(+), 114 deletions(-)
diff --git a/regression-test/data/inverted_index_p0/load/test_stream_load.out
b/regression-test/data/inverted_index_p0/load/test_stream_load.out
index 5723e4218d2..c09ee5451bc 100644
--- a/regression-test/data/inverted_index_p0/load/test_stream_load.out
+++ b/regression-test/data/inverted_index_p0/load/test_stream_load.out
@@ -2,44 +2,16 @@
-- !sql_1 --
davidjhulse/davesbingrewardsbot
davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-- !sql_1 --
davidjhulse/davesbingrewardsbot
davidjhulse/davesbingrewardsbot
+
+-- !sql_1 --
davidjhulse/davesbingrewardsbot
davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
-davidjhulse/davesbingrewardsbot
+
+-- !sql_1 --
davidjhulse/davesbingrewardsbot
davidjhulse/davesbingrewardsbot
diff --git
a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
index f29ff3b3512..53ee5d546c2 100644
--- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
@@ -16,7 +16,7 @@
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
-suite("test_stream_load_with_inverted_index_p0", "nonCurrent") {
+suite("test_stream_load_with_inverted_index_p0", "p0") {
def set_be_config = { key, value ->
def backendId_to_backendIP = [:]
@@ -63,88 +63,52 @@ suite("test_stream_load_with_inverted_index_p0",
"nonCurrent") {
}
}
- boolean disableAutoCompaction = true
- boolean has_update_be_config = false
- try {
- String backend_id;
- def backendId_to_backendIP = [:]
- def backendId_to_backendHttpPort = [:]
- getBackendIpHttpPort(backendId_to_backendIP,
backendId_to_backendHttpPort);
-
- backend_id = backendId_to_backendIP.keySet()[0]
- def (code, out, err) =
show_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id))
-
- logger.info("Show config: code=" + code + ", out=" + out + ", err=" +
err)
- assertEquals(code, 0)
- def configList = parseJson(out.trim())
- assert configList instanceof List
-
- for (Object ele in (List) configList) {
- assert ele instanceof List<String>
- if (((List<String>) ele)[0] == "disable_auto_compaction") {
- disableAutoCompaction = Boolean.parseBoolean(((List<String>)
ele)[2])
- }
- }
- set_be_config.call("disable_auto_compaction", "true")
- has_update_be_config = true
-
- def test = { format ->
- sql """ DROP TABLE IF EXISTS ${tableName}; """
- sql """
- CREATE TABLE ${tableName} (
- k bigint,
- v variant,
- INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
- ) ENGINE=OLAP
- DUPLICATE KEY(`k`)
- COMMENT 'OLAP'
- DISTRIBUTED BY HASH(`k`) BUCKETS 10
- PROPERTIES ( "replication_num" = "1",
"inverted_index_storage_format" = ${format});
- """
-
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
-
-
- def tablets = sql_return_maparray """ show tablets from
${tableName}; """
-
- for (def tablet in tablets) {
- String tablet_id = tablet.TabletId
- backend_id = tablet.BackendId
- (code, out, err) =
calc_file_crc_on_tablet(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), tablet_id)
- logger.info("Run calc file: code=" + code + ", out=" + out +
", err=" + err)
- assertEquals(code, 0)
- def resultJson = parseJson(out.trim())
- assertEquals(resultJson.start_version, "0")
- assertEquals(resultJson.end_version, "11")
- assertEquals(resultJson.rowset_count, "11")
- }
- qt_sql_1 """
- select cast(v["repo"]["name"] as string) from ${tableName} where
cast(v["repo"]["name"] as string) match_phrase_prefix "davesbingrewardsbot";
- """
-
- sql """ DROP TABLE IF EXISTS ${tableName}; """
- }
-
- set_be_config("inverted_index_ram_dir_enable", "true")
- // test.call("V1")
- test.call("V2")
- set_be_config("inverted_index_ram_dir_enable", "false")
- // test.call("V1")
- test.call("V2")
- set_be_config("inverted_index_ram_dir_enable", "true")
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ def test = { format ->
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
+ sql """
+ CREATE TABLE ${tableName} (
+ k bigint,
+ v variant,
+ INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" =
"english") COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k`) BUCKETS 10
+ PROPERTIES ( "replication_num" = "1",
"inverted_index_storage_format" = "${format}", "disable_auto_compaction" =
"true");
+ """
+
+ load_json_data.call(tableName, """${getS3Url() +
'/regression/gharchive.m/2015-01-01-0.json'}""")
- } finally {
- if (has_update_be_config) {
- set_be_config.call("disable_auto_compaction",
disableAutoCompaction.toString())
+ def tablets = sql_return_maparray """ show tablets from ${tableName};
"""
+
+ for (def tablet in tablets) {
+ String tablet_id = tablet.TabletId
+ backend_id = tablet.BackendId
+ (code, out, err) =
calc_file_crc_on_tablet(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), tablet_id)
+ logger.info("Run calc file: code=" + code + ", out=" + out + ",
err=" + err)
+ assertEquals(code, 0)
+ def resultJson = parseJson(out.trim())
+ assertEquals(resultJson.start_version, "0")
+ assertEquals(resultJson.end_version, "2")
+ assertEquals(resultJson.rowset_count, "2")
}
+ qt_sql_1 """
+ select cast(v["repo"]["name"] as string) from ${tableName} where
cast(v["repo"]["name"] as string) match_phrase_prefix "davesbingrewardsbot";
+ """
+
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
}
+
+ set_be_config("inverted_index_ram_dir_enable", "true")
+ test.call("V1")
+ test.call("V2")
+ set_be_config("inverted_index_ram_dir_enable", "false")
+ test.call("V1")
+ test.call("V2")
+ set_be_config("inverted_index_ram_dir_enable", "true")
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
