This is an automated email from the ASF dual-hosted git repository. shuke987 pushed a commit to branch fix-index-change-wait-helper in repository https://gitbox.apache.org/repos/asf/doris.git
commit 25e5b66c45a0fa49bba05ca598748a11792bd404 Author: shuke <[email protected]> AuthorDate: Thu Jul 16 15:07:36 2026 +0800 [fix](regression) make index change waits operation scoped --- regression-test/plugins/plugin_index_change.groovy | 55 +++---------- .../test_cloud_build_index_basic.groovy | 37 +++++---- .../test_cloud_build_index_update.groovy | 22 +++--- .../index_change/test_add_drop_index.groovy | 92 +++++++++++++--------- .../test_add_drop_index_ignore_case_column.groovy | 41 +++------- .../test_add_drop_index_repeatly.groovy | 38 ++------- .../test_add_drop_index_with_data.groovy | 54 +++++-------- .../test_add_drop_index_with_delete_data.groovy | 41 +++------- .../test_drop_index_on_partition.groovy | 9 +++ .../index_change/test_index_change_3.groovy | 30 ++++--- .../index_change/test_index_change_5.groovy | 20 +++-- .../test_index_change_with_full_compaction.groovy | 22 ++++-- .../suites/variant_p0/with_index/load.groovy | 23 +++--- 13 files changed, 211 insertions(+), 273 deletions(-) diff --git a/regression-test/plugins/plugin_index_change.groovy b/regression-test/plugins/plugin_index_change.groovy index d3741b6cc99..69595df2c3a 100644 --- a/regression-test/plugins/plugin_index_change.groovy +++ b/regression-test/plugins/plugin_index_change.groovy @@ -28,42 +28,10 @@ Suite.metaClass.snapshot_build_index_job_ids = { table_name -> return alter_res.collect { it[0].toString() }.toSet() } -// Compatibility helper for legacy callers where a build-index job is optional. -// New explicit build/delete paths must snapshot job ids before the operation and -// use wait_for_new_build_index_jobs_finish instead. -Suite.metaClass.wait_for_last_build_index_finish = { table_name, OpTimeout -> - def finished = false - def alter_res = [] - for (int t = 0; t <= OpTimeout; t += delta_time) { - alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}" - ORDER BY CreateTime DESC LIMIT 1;""" - if (alter_res.isEmpty()) { - logger.info(table_name + " has no build index job") - finished = true - break - } - if (alter_res.any { it[7] == "CANCELLED" }) { - logger.info(table_name + " latest build index job failed, detail: " + alter_res) - assertTrue(false) - } - if (alter_res.every { it[7] == "FINISHED" }) { - logger.info(table_name + " latest build index job finished, detail: " + alter_res) - finished = true - break - } - if (t >= OpTimeout) { - break - } - sleep(delta_time) - } - assertTrue(finished, "wait for latest build index finish timeout, latest result: ${alter_res}") -} - Suite.metaClass.wait_for_new_build_index_jobs_finish = { table_name, OpTimeout, previous_job_ids -> assertTrue(previous_job_ids != null, "previous build index job ids must be snapshotted before the operation") def finished = false def alter_res = [] - def confirmed_finished_job_ids = null for (int t = 0; t <= OpTimeout; t += delta_time) { def all_jobs = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}" ORDER BY CreateTime DESC;""" @@ -71,18 +39,12 @@ Suite.metaClass.wait_for_new_build_index_jobs_finish = { table_name, OpTimeout, if (!alter_res.isEmpty() && alter_res.any { it[7] == "CANCELLED" }) { logger.info(table_name + " build index job failed, detail: " + alter_res) - assertTrue(false) + assertTrue(false, "build index job cancelled, result: ${alter_res}") } if (!alter_res.isEmpty() && alter_res.every { it[7] == "FINISHED" }) { - def finished_job_ids = alter_res.collect { it[0].toString() }.toSet() - if (finished_job_ids == confirmed_finished_job_ids) { - logger.info(table_name + " build index jobs finished, detail: " + alter_res) - finished = true - break - } - confirmed_finished_job_ids = finished_job_ids - } else { - confirmed_finished_job_ids = null + logger.info(table_name + " build index jobs finished, detail: " + alter_res) + finished = true + break } if (t >= OpTimeout) { break @@ -92,6 +54,15 @@ Suite.metaClass.wait_for_new_build_index_jobs_finish = { table_name, OpTimeout, assertTrue(finished, "wait for build index finish timeout, latest result: ${alter_res}") } +// Run an operation that must create one or more IndexChangeJobs and wait for +// every job created by that operation. SHOW BUILD INDEX history is retained, so +// the pre-operation snapshot is required to isolate the current operation. +Suite.metaClass.run_index_change_job_and_wait = { table_name, OpTimeout, Closure operation -> + def previous_job_ids = snapshot_build_index_job_ids(table_name) + operation.call() + wait_for_new_build_index_jobs_finish(table_name, OpTimeout, previous_job_ids) +} + Suite.metaClass.build_index_on_table = {index_name, table_name -> if (isCloudMode()) { sql """build index on ${table_name}""" diff --git a/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_basic.groovy b/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_basic.groovy index 3fd71c8ca52..6f77af6fdcf 100644 --- a/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_basic.groovy +++ b/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_basic.groovy @@ -57,12 +57,14 @@ suite("test_cloud_build_index_basic"){ sql """create index idx1 on test_cloud_build_index_basic_table(address) using inverted;""" check_inverted_index_filter_rows("select * from test_cloud_build_index_basic_table where address='dddd'", 0) - build_index_on_table("idx1", tabName1) - wait_for_last_build_index_finish(tabName1, timeout) + run_index_change_job_and_wait(tabName1, timeout) { + build_index_on_table("idx1", tabName1) + } check_inverted_index_filter_rows("select * from test_cloud_build_index_basic_table where address='dddd'", 4) - sql "drop index idx1 on test_cloud_build_index_basic_table" - wait_for_last_build_index_finish(tabName1, timeout) + run_index_change_job_and_wait(tabName1, timeout) { + sql "drop index idx1 on test_cloud_build_index_basic_table" + } check_inverted_index_filter_rows("select * from test_cloud_build_index_basic_table where address='dddd'", 0) // 2 string type, ngram index @@ -70,12 +72,14 @@ suite("test_cloud_build_index_basic"){ sql """create index idx2 on test_cloud_build_index_basic_table(address) using ngram_bf properties("gram_size"="3", "bf_size"="256");""" check_bf_index_filter_rows("select * from test_cloud_build_index_basic_table where address like '%sdf%'", 0) - build_index_on_table("idx2", tabName1) - wait_for_last_build_index_finish(tabName1, timeout) + run_index_change_job_and_wait(tabName1, timeout) { + build_index_on_table("idx2", tabName1) + } check_bf_index_filter_rows("select * from test_cloud_build_index_basic_table where address like '%sdf%'", 5) - sql "drop index idx2 on test_cloud_build_index_basic_table" - wait_for_last_build_index_finish(tabName1, timeout) + run_index_change_job_and_wait(tabName1, timeout) { + sql "drop index idx2 on test_cloud_build_index_basic_table" + } check_bf_index_filter_rows("select * from test_cloud_build_index_basic_table where address like '%sdf%'", 0) // test add column/reset varchar len @@ -121,18 +125,21 @@ suite("test_cloud_build_index_basic"){ check_inverted_index_filter_rows("select * from test_cloud_light_sc_table where content='jj1'", 0) sql """alter table test_cloud_light_sc_table modify column address varchar(400);""" - sql """build index on test_cloud_light_sc_table;""" - wait_for_last_build_index_finish("test_cloud_light_sc_table", timeout) + run_index_change_job_and_wait("test_cloud_light_sc_table", timeout) { + sql """build index on test_cloud_light_sc_table;""" + } check_inverted_index_filter_rows("select * from test_cloud_light_sc_table where address='eeeee'", 4) check_inverted_index_filter_rows("select * from test_cloud_light_sc_table where content='jj1'", 4) - sql """drop index idx1 on test_cloud_light_sc_table;""" - wait_for_last_build_index_finish("test_cloud_light_sc_table", timeout) + run_index_change_job_and_wait("test_cloud_light_sc_table", timeout) { + sql """drop index idx1 on test_cloud_light_sc_table;""" + } check_inverted_index_filter_rows("select * from test_cloud_light_sc_table where address='eeeee'", 0) - sql """drop index idx2 on test_cloud_light_sc_table;""" - wait_for_last_build_index_finish("test_cloud_light_sc_table", timeout) + run_index_change_job_and_wait("test_cloud_light_sc_table", timeout) { + sql """drop index idx2 on test_cloud_light_sc_table;""" + } check_inverted_index_filter_rows("select * from test_cloud_light_sc_table where content='jj1'", 0) qt_select_sc_output "select * from test_cloud_light_sc_table order by user_id,username,age,address,content" -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_update.groovy b/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_update.groovy index 31c25655a01..3ebd4599329 100644 --- a/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_update.groovy +++ b/regression-test/suites/inverted_index_p0/cloud_build_index/test_cloud_build_index_update.groovy @@ -88,13 +88,15 @@ suite("test_cloud_build_index_update") { check_inverted_index_filter_rows("select * from test_cloud_build_idx_uq_table where address='hhhhh'" + " order by user_id,username,age,address", 0) - sql """build index on test_cloud_build_idx_uq_table""" - wait_for_last_build_index_finish("test_cloud_build_idx_uq_table", timeout) + run_index_change_job_and_wait("test_cloud_build_idx_uq_table", timeout) { + sql """build index on test_cloud_build_idx_uq_table""" + } check_inverted_index_filter_rows("select * from test_cloud_build_idx_uq_table where address='hhhhh'" + " order by user_id,username,age,address", 12) - sql """drop index idx1 on test_cloud_build_idx_uq_table""" - wait_for_last_build_index_finish("test_cloud_build_idx_uq_table", timeout) + run_index_change_job_and_wait("test_cloud_build_idx_uq_table", timeout) { + sql """drop index idx1 on test_cloud_build_idx_uq_table""" + } check_inverted_index_filter_rows("select * from test_cloud_build_idx_uq_table where address='hhhhh'" + " order by user_id,username,age,address", 0) @@ -135,8 +137,9 @@ suite("test_cloud_build_index_update") { sql """create index idx1 on test_cloud_build_idx_del_tab(address) using inverted;""" - build_index_on_table("idx1", "test_cloud_build_idx_del_tab") - wait_for_last_build_index_finish("test_cloud_build_idx_del_tab", timeout) + run_index_change_job_and_wait("test_cloud_build_idx_del_tab", timeout) { + build_index_on_table("idx1", "test_cloud_build_idx_del_tab") + } check_inverted_index_filter_rows("select * from test_cloud_build_idx_del_tab" + " where address='eeeee' order by user_id,username,age,address", 12) @@ -144,8 +147,9 @@ suite("test_cloud_build_index_update") { qt_select_del_tab_ret3 """select count(1) from test_cloud_build_idx_del_tab where age=23;""" qt_select_del_tab_ret4 """select count(1) from test_cloud_build_idx_del_tab where age!=23;""" - sql """drop index idx1 on test_cloud_build_idx_del_tab;""" - wait_for_last_build_index_finish("test_cloud_build_idx_del_tab", timeout) + run_index_change_job_and_wait("test_cloud_build_idx_del_tab", timeout) { + sql """drop index idx1 on test_cloud_build_idx_del_tab;""" + } check_inverted_index_filter_rows("select * from test_cloud_build_idx_del_tab" + " where address='eeeee' order by user_id,username,age,address", 0) @@ -153,4 +157,4 @@ suite("test_cloud_build_index_update") { qt_select_del_tab_ret5 """select count(1) from test_cloud_build_idx_del_tab where age=23;""" qt_select_del_tab_ret6 """select count(1) from test_cloud_build_idx_del_tab where age!=23;""" -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index.groovy b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index.groovy index ea49d57154a..c55874eb314 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index.groovy @@ -47,8 +47,9 @@ suite("test_add_drop_index", "inverted_index"){ // case1: create index for int colume // case1.0 create index sql "create index age_idx on ${indexTbName1}(tearchComment) using inverted" - build_index_on_table("age_idx", indexTbName1) - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("age_idx", indexTbName1) + } def show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) @@ -74,8 +75,9 @@ suite("test_add_drop_index", "inverted_index"){ // case1.3 create duplicate different index for one colume with different name sql "create index age_idx_diff on ${indexTbName1}(`tearchComment`) using NGRAM_BF" if (isCloudMode()) { - build_index_on_table("age_idx_diff", indexTbName1) - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("age_idx_diff", indexTbName1) + } } else { wait_for_last_col_change_finish(indexTbName1, timeout) } @@ -85,13 +87,15 @@ suite("test_add_drop_index", "inverted_index"){ assertEquals(show_result[1][2], "age_idx_diff") // case1.4 drop index - def drop_result = sql """ - ALTER TABLE ${indexTbName1} - drop index age_idx, - drop index age_idx_diff; - """ + def drop_result + run_index_change_job_and_wait(indexTbName1, timeout) { + drop_result = sql """ + ALTER TABLE ${indexTbName1} + drop index age_idx, + drop index age_idx_diff; + """ + } logger.info("drop index age_idx and age_idx_diff on " + indexTbName1 + "; result: " + drop_result) - wait_for_last_build_index_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" assertEquals(show_result.size(), 0) @@ -107,12 +111,14 @@ suite("test_add_drop_index", "inverted_index"){ // case2: create index for date colume sql "create index date_idx on ${indexTbName1}(`registDate`) using inverted" - build_index_on_table("date_idx", indexTbName1) - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("date_idx", indexTbName1) + } show_result = sql "show index from ${indexTbName1}" assertEquals(show_result.size(), 1) - sql "drop index date_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index date_idx on ${indexTbName1}" + } // case3: create string inverted index for int colume def create_string_index_on_int_colume_result = "fail" @@ -127,14 +133,16 @@ suite("test_add_drop_index", "inverted_index"){ // case4: create default inverted index for varchar coulume sql "create index name_idx on ${indexTbName1}(`name`) using inverted" - build_index_on_table("name_idx", indexTbName1) - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("name_idx", indexTbName1) + } show_result = sql "show index from ${indexTbName1}" assertTrue(show_result[0][2] == "name_idx" && show_result[0][10] == "INVERTED") logger.info("create index name_idx for " + indexTbName1 + "(`name`)") logger.info("show index result: " + show_result) - sql "drop index name_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index name_idx on ${indexTbName1}" + } // case5: create none inverted index for char colume sql "create index name_idx on ${indexTbName1}(`name`) USING INVERTED PROPERTIES(\"parser\"=\"none\")" @@ -143,11 +151,9 @@ suite("test_add_drop_index", "inverted_index"){ assertEquals(show_result[0][10], "INVERTED") logger.info("create index name_idx for " + indexTbName1 + "(`name`) USING INVERTED PROPERTIES(\"parser\"=\"none\")") logger.info("show index result: " + show_result) - def previous_job_ids = isCloudMode() ? snapshot_build_index_job_ids(indexTbName1) : null - sql "drop index name_idx on ${indexTbName1}" - wait_for_last_schema_change_finish(indexTbName1, timeout) - if (isCloudMode()) { - wait_for_new_build_index_jobs_finish(indexTbName1, timeout, previous_job_ids) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index name_idx on ${indexTbName1}" + wait_for_last_schema_change_finish(indexTbName1, timeout) } // case5：create simple inverted index for char colume @@ -157,8 +163,9 @@ suite("test_add_drop_index", "inverted_index"){ assertTrue(show_result[0][2] == "studentInfo_idx" && show_result[0][10] == "INVERTED") logger.info("create index studentInfo_idx for " + indexTbName1 + "(`studentInfo`) USING INVERTED PROPERTIES(\"parser\"=\"english\")") logger.info("show index result: " + show_result) - sql "drop index studentInfo_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index studentInfo_idx on ${indexTbName1}" + } // case6: create standard inverted index for text colume sql "create index tearchComment_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")" @@ -167,8 +174,9 @@ suite("test_add_drop_index", "inverted_index"){ assertTrue(show_result[0][2] == "tearchComment_idx" && show_result[0][10] == "INVERTED") logger.info("create index tearchComment_idx for " + indexTbName1 + "(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")") logger.info("show index result: " + show_result) - sql "drop index tearchComment_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx on ${indexTbName1}" + } // case7: drop not exist index def drop_no_exist_index_result = "fail" @@ -184,37 +192,43 @@ suite("test_add_drop_index", "inverted_index"){ // case8.0: create, drop, create same index with same name sql "create index tearchComment_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")" wait_for_last_col_change_finish(indexTbName1, timeout) - sql "drop index tearchComment_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx on ${indexTbName1}" + } sql "create index tearchComment_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")" wait_for_last_col_change_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" assertTrue(show_result[0][2] == "tearchComment_idx" && show_result[0][10] == "INVERTED") - sql "drop index tearchComment_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx on ${indexTbName1}" + } // case8.1: create, drop, create other index with same name sql "create index tearchComment_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")" wait_for_last_col_change_finish(indexTbName1, timeout) - sql "drop index tearchComment_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx on ${indexTbName1}" + } sql "create index tearchComment_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"none\")" wait_for_last_col_change_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" assertTrue(show_result[0][2] == "tearchComment_idx" && show_result[0][10] == "INVERTED") - sql "drop index tearchComment_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx on ${indexTbName1}" + } // case8.2: create, drop, create same index with other name sql "create index tearchComment_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")" wait_for_last_col_change_finish(indexTbName1, timeout) - sql "drop index tearchComment_idx on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx on ${indexTbName1}" + } sql "create index tearchComment_idx_2 on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")" wait_for_last_col_change_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" assertTrue(show_result[0][2] == "tearchComment_idx_2" && show_result[0][10] == "INVERTED") - sql "drop index tearchComment_idx_2 on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index tearchComment_idx_2 on ${indexTbName1}" + } } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_ignore_case_column.groovy b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_ignore_case_column.groovy index caa21946fbd..d2f521d9b38 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_ignore_case_column.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_ignore_case_column.groovy @@ -21,30 +21,6 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ // prepare test table def timeout = 60000 - def delta_time = 1000 - def alter_res = "null" - def useTime = 0 - - def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> - for(int t = delta_time; t <= OpTimeout; t += delta_time){ - alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" - def expected_finished_num = alter_res.size(); - def finished_num = 0; - for (int i = 0; i < expected_finished_num; i++) { - logger.info(table_name + " build index job state: " + alter_res[i][7] + i) - if (alter_res[i][7] == "FINISHED") { - ++finished_num; - } - } - if (finished_num == expected_finished_num) { - logger.info(table_name + " all build index jobs finished, detail: " + alter_res) - break - } - useTime = t - sleep(delta_time) - } - assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") - } def indexTbName1 = "test_add_drop_inverted_index4" @@ -113,8 +89,9 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_last_col_change_finish(indexTbName1, timeout) if (!isCloudMode()) { - sql "build index idx_desc on ${indexTbName1}" - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "build index idx_desc on ${indexTbName1}" + } } // show index after add index @@ -150,8 +127,9 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ // drop index // add index on column description - sql "drop index idx_desc on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index idx_desc on ${indexTbName1}" + } // query rows where description match 'desc' without index select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" @@ -181,8 +159,9 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ sql "create index idx_desc on ${indexTbName1}(DESCRIPTION) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_last_col_change_finish(indexTbName1, timeout) if (!isCloudMode()) { - sql "build index idx_desc on ${indexTbName1}" - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "build index idx_desc on ${indexTbName1}" + } } // query rows where description match 'desc' @@ -208,4 +187,4 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ assertEquals(select_result[0][0], 2) assertEquals(select_result[0][1], "name2") assertEquals(select_result[0][2], "desc 2") -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_repeatly.groovy b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_repeatly.groovy index 560f31152da..28375bdcbfb 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_repeatly.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_repeatly.groovy @@ -19,33 +19,6 @@ suite("test_add_drop_index_repeatly"){ sql "set enable_add_index_for_new_data = true" def timeout = 300000 - def delta_time = 1000 - def alter_res = "null" - def useTime = 0 - - def wait_for_last_build_index_on_table_finish = { table_name, OpTimeout -> - for(int t = delta_time; t <= OpTimeout; t += delta_time){ - alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}" ORDER BY JobId """ - - if (alter_res.size() == 0) { - logger.info(table_name + " last index job finished") - return "SKIPPED" - } - if (alter_res.size() > 0) { - def last_job_state = alter_res[alter_res.size()-1][7]; - if (last_job_state == "FINISHED" || last_job_state == "CANCELLED") { - sleep(10000) // wait change table state to normal - logger.info(table_name + " last index job finished, state: " + last_job_state + ", detail: " + alter_res) - return last_job_state; - } - } - useTime = t - sleep(delta_time) - } - logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res) - assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout") - return "wait_timeout" - } def tbl = 'test_add_drop_index_repeatly' sql """ DROP TABLE IF EXISTS ${tbl} """ @@ -70,11 +43,12 @@ suite("test_add_drop_index_repeatly"){ sql """ create index idx_k2 on ${tbl}(k2) using inverted """ // build index - build_index_on_table("idx_k2", tbl) - def state = wait_for_last_build_index_on_table_finish(tbl, timeout) - assertEquals(state, "FINISHED") + run_index_change_job_and_wait(tbl, timeout) { + build_index_on_table("idx_k2", tbl) + } - sql """ drop index idx_k2 on ${tbl} """ - wait_for_last_build_index_finish(tbl, timeout) + run_index_change_job_and_wait(tbl, timeout) { + sql """ drop index idx_k2 on ${tbl} """ + } } } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_data.groovy b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_data.groovy index 79e1d11bbf8..09461d5b18f 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_data.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_data.groovy @@ -19,33 +19,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ // prepare test table def timeout = 60000 - def delta_time = 1000 - def alter_res = "null" - def useTime = 0 sql "set enable_add_index_for_new_data = true" - def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> - for(int t = delta_time; t <= OpTimeout; t += delta_time){ - alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" - def expected_finished_num = alter_res.size(); - def finished_num = 0; - for (int i = 0; i < expected_finished_num; i++) { - logger.info(table_name + " build index job state: " + alter_res[i][7] + i) - if (alter_res[i][7] == "FINISHED") { - ++finished_num; - } - } - if (finished_num == expected_finished_num) { - logger.info(table_name + " all build index jobs finished, detail: " + alter_res) - break - } - useTime = t - sleep(delta_time) - } - assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") - } - def indexTbName1 = "test_add_drop_inverted_index2" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -114,8 +90,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ wait_for_last_col_change_finish(indexTbName1, timeout) if(!isCloudMode()) { - build_index_on_table("idx_desc", indexTbName1) - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("idx_desc", indexTbName1) + } } // show index after add index @@ -151,8 +128,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ // drop index // add index on column description - sql "drop index idx_desc on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index idx_desc on ${indexTbName1}" + } // query rows where description match 'desc', should fail without index select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" @@ -179,8 +157,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ assertEquals(select_result[0][2], "desc 2") // drop idx_id index - sql "drop index idx_id on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index idx_id on ${indexTbName1}" + } // show index of create table show_result = sql "show index from ${indexTbName1}" @@ -203,8 +182,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ assertEquals(select_result[0][2], "desc 2") // drop idx_name index - sql "drop index idx_name on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index idx_name on ${indexTbName1}" + } // query rows where name match 'name1' without index select_result = sql "select * from ${indexTbName1} where name match 'name1'" @@ -222,8 +202,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_last_col_change_finish(indexTbName1, timeout) if (!isCloudMode()) { - build_index_on_table("idx_desc", indexTbName1) - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("idx_desc", indexTbName1) + } } // query rows where description match 'desc' @@ -261,8 +242,9 @@ suite("test_add_drop_index_with_data", "inverted_index"){ logger.info("show index from " + indexTbName1 + " result: " + show_result) assertEquals(show_result.size(), 3) - build_index_on_table("idx_name", indexTbName1) - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + build_index_on_table("idx_name", indexTbName1) + } // query rows where name match 'name1' select_result = sql "select * from ${indexTbName1} where name match 'name1'" diff --git a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_delete_data.groovy b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_delete_data.groovy index 120dfa2cd09..6a057f8a53a 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_delete_data.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_with_delete_data.groovy @@ -19,33 +19,9 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ // prepare test table def timeout = 60000 - def delta_time = 1000 - def alter_res = "null" - def useTime = 0 sql "set enable_add_index_for_new_data = true" - def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> - for(int t = delta_time; t <= OpTimeout; t += delta_time){ - alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" - def expected_finished_num = alter_res.size(); - def finished_num = 0; - for (int i = 0; i < expected_finished_num; i++) { - logger.info(table_name + " build index job state: " + alter_res[i][7] + i) - if (alter_res[i][7] == "FINISHED") { - ++finished_num; - } - } - if (finished_num == expected_finished_num) { - logger.info(table_name + " all build index jobs finished, detail: " + alter_res) - break - } - useTime = t - sleep(delta_time) - } - assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") - } - def indexTbName1 = "test_add_drop_inverted_index3" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -121,8 +97,9 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ wait_for_last_col_change_finish(indexTbName1, timeout) if (!isCloudMode()) { - sql "build index idx_desc on ${indexTbName1}" - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "build index idx_desc on ${indexTbName1}" + } } // show index after add index @@ -185,8 +162,9 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ assertEquals(select_result[1][2], "desc ok world test") // drop index - sql "drop index idx_desc on ${indexTbName1}" - wait_for_last_build_index_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "drop index idx_desc on ${indexTbName1}" + } // query rows where description match 'desc' without index select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" assertEquals(select_result.size(), 4) @@ -230,8 +208,9 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ wait_for_last_col_change_finish(indexTbName1, timeout) if (!isCloudMode()) { - sql "build index idx_desc on ${indexTbName1}" - wait_for_build_index_on_partition_finish(indexTbName1, timeout) + run_index_change_job_and_wait(indexTbName1, timeout) { + sql "build index idx_desc on ${indexTbName1}" + } } // show index after add index @@ -274,4 +253,4 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ // query rows where description match 'ok' select_result = sql "select * from ${indexTbName1} where description match 'ok' order by id" assertEquals(select_result.size(), 0) -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy b/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy index b7e33a6a7d8..8489f68fa64 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy @@ -45,6 +45,9 @@ suite("test_drop_index_on_partition", "inverted_index") { // create inverted index sql "CREATE INDEX idx_v2 ON ${tableName1}(v2) USING INVERTED" wait_for_last_col_change_finish(tableName1, timeout) + run_index_change_job_and_wait(tableName1, timeout) { + build_index_on_table("idx_v2", tableName1) + } // verify index exists def show_idx = sql "SHOW INDEX FROM ${tableName1}" @@ -103,6 +106,9 @@ suite("test_drop_index_on_partition", "inverted_index") { sql "CREATE INDEX idx_v1 ON ${tableName2}(v1) USING INVERTED" wait_for_last_col_change_finish(tableName2, timeout) + run_index_change_job_and_wait(tableName2, timeout) { + build_index_on_table("idx_v1", tableName2) + } previous_job_ids = snapshot_build_index_job_ids(tableName2) @@ -218,6 +224,9 @@ suite("test_drop_index_on_partition", "inverted_index") { sql "CREATE INDEX idx_v1 ON ${tableName5}(v1) USING INVERTED" wait_for_last_col_change_finish(tableName5, timeout) + run_index_change_job_and_wait(tableName5, timeout) { + build_index_on_table("idx_v1", tableName5) + } previous_job_ids = snapshot_build_index_job_ids(tableName5) diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy index f6048fc387c..6de7be2c0dd 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy @@ -75,15 +75,18 @@ suite("test_index_change_3") { sql """ CREATE INDEX idx_city ON ${tableName}(`city`) using inverted properties("support_phrase" = "true", "parser" = "english", "lower_case" = "true") """ wait_for_last_col_change_finish(tableName, timeout) if (!isCloudMode()) { - build_index_on_table("idx_city", tableName) - wait_for_last_build_index_finish(tableName, timeout) + run_index_change_job_and_wait(tableName, timeout) { + build_index_on_table("idx_city", tableName) + } } // drop inverted index idx_user_id, idx_note - sql """ DROP INDEX idx_city ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) - sql """ DROP INDEX idx_note ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_city ON ${tableName} """ + } + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_note ON ${tableName} """ + } def show_result = sql "show index from ${tableName}" logger.info("show index from " + tableName + " result: " + show_result) @@ -150,15 +153,18 @@ suite("test_index_change_3") { wait_for_last_col_change_finish(tableName, timeout) if (!isCloudMode()) { - build_index_on_table("idx_city", tableName) - wait_for_last_build_index_finish(tableName, timeout) + run_index_change_job_and_wait(tableName, timeout) { + build_index_on_table("idx_city", tableName) + } } // drop inverted index idx_user_id, idx_note - sql """ DROP INDEX idx_city ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) - sql """ DROP INDEX idx_note ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_city ON ${tableName} """ + } + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_note ON ${tableName} """ + } show_result = sql "show index from ${tableName}" logger.info("show index from " + tableName + " result: " + show_result) diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy index ea236eba0e1..9bca2219b56 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy @@ -72,10 +72,12 @@ suite("test_index_change_5") { qt_select6 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ // drop inverted index idx_user_id, idx_note - sql """ DROP INDEX idx_user_id ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) - sql """ DROP INDEX idx_note ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_user_id ON ${tableName} """ + } + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_note ON ${tableName} """ + } def show_result = sql "show index from ${tableName}" logger.info("show index from " + tableName + " result: " + show_result) @@ -137,10 +139,12 @@ suite("test_index_change_5") { qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ // drop inverted index idx_user_id, idx_note - sql """ DROP INDEX idx_user_id ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) - sql """ DROP INDEX idx_note ON ${tableName} """ - wait_for_last_build_index_finish(tableName, timeout) + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_user_id ON ${tableName} """ + } + run_index_change_job_and_wait(tableName, timeout) { + sql """ DROP INDEX idx_note ON ${tableName} """ + } show_result = sql "show index from ${tableName}" logger.info("show index from " + tableName + " result: " + show_result) diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy index 5067f65c01f..d94ce2a346f 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy @@ -132,12 +132,22 @@ suite("test_index_change_with_full_compaction") { // build index - build_index_on_table("idx_user_id", tableName) - wait_for_last_build_index_finish(tableName, timeout) - build_index_on_table("idx_date", tableName) - wait_for_last_build_index_finish(tableName, timeout) - build_index_on_table("idx_city", tableName) - wait_for_last_build_index_finish(tableName, timeout) + if (isCloudMode()) { + // Cloud BUILD INDEX works at table scope and builds all indexes. + run_index_change_job_and_wait(tableName, timeout) { + build_index_on_table("idx_user_id", tableName) + } + } else { + run_index_change_job_and_wait(tableName, timeout) { + build_index_on_table("idx_user_id", tableName) + } + run_index_change_job_and_wait(tableName, timeout) { + build_index_on_table("idx_date", tableName) + } + run_index_change_job_and_wait(tableName, timeout) { + build_index_on_table("idx_city", tableName) + } + } // trigger compactions for all tablets in ${tableName} trigger_and_wait_compaction(tableName, "full") diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy index 86197285d41..a46ef8eb71d 100644 --- a/regression-test/suites/variant_p0/with_index/load.groovy +++ b/regression-test/suites/variant_p0/with_index/load.groovy @@ -44,25 +44,24 @@ suite("regression_test_variant_with_index", "p0"){ // sql "truncate table ${table_name}" sql """insert into var_with_index values(1, '{"a1" : 0, "b1": 3}', 'hello world'), (2, '{"a2" : 123}', 'world'),(3, '{"a3" : 123}', 'hello world')""" sql """insert into var_with_index values(4, '{"b1" : 0, "b2": 3}', 'hello world'), (5, '{"b2" : 123}', 'world'),(6, '{"b3" : 123}', 'hello world')""" - def previous_job_ids = isCloudMode() ? snapshot_build_index_job_ids(table_name) : null - def drop_result = sql """ - ALTER TABLE var_with_index - drop index idx - """ - logger.info("drop index " + "${table_name}" + "; result: " + drop_result) - wait_for_last_schema_change_finish(table_name, timeout) - if (isCloudMode()) { - wait_for_new_build_index_jobs_finish(table_name, timeout, previous_job_ids) + def drop_result + run_index_change_job_and_wait(table_name, timeout) { + drop_result = sql """ + ALTER TABLE var_with_index + drop index idx + """ + wait_for_last_schema_change_finish(table_name, timeout) } + logger.info("drop index " + "${table_name}" + "; result: " + drop_result) def show_result = sql "show index from ${table_name}" assertEquals(show_result.size(), 0) qt_sql_inv4 """select v["a1"] from ${table_name} where cast(v['a1'] as int) = 0""" qt_sql_inv5 """select * from ${table_name} order by k""" sql "create index inv_idx on ${table_name}(`inv`) using inverted" wait_for_last_col_change_finish(table_name, timeout) - previous_job_ids = snapshot_build_index_job_ids(table_name) - build_index_on_table("inv_idx", table_name) - wait_for_new_build_index_jobs_finish(table_name, timeout, previous_job_ids) + run_index_change_job_and_wait(table_name, timeout) { + build_index_on_table("inv_idx", table_name) + } show_result = sql "show index from ${table_name}" assertEquals(show_result.size(), 1) sql """insert into var_with_index values(7, '{"a1" : 0, "b1": 3}', 'hello world'), (8, '{"a2" : 123}', 'world'),(9, '{"a3" : 123}', 'hello world')""" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
