This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6997929b2fd [test](ngram) make ngram index test case robust (#57943)
6997929b2fd is described below
commit 6997929b2fdd0689778ca3e80bef7badf2c2031b
Author: Jack <[email protected]>
AuthorDate: Wed Nov 12 18:13:13 2025 +0800
[test](ngram) make ngram index test case robust (#57943)
---
.../test_ngram_bloomfilter_index_change.groovy | 213 +++++++++++----------
1 file changed, 112 insertions(+), 101 deletions(-)
diff --git
a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
index aebcdfbfb0b..b5d0b9508f4 100644
--- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
+++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
@@ -23,6 +23,64 @@ suite("test_ngram_bloomfilter_index_change") {
def alter_res = "null"
def useTime = 0
+ // Helper functions to fetch profile via HTTP API
+ def getProfileList = {
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
+ conn.setRequestMethod("GET")
+ def encoding =
Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" :
context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ return conn.getInputStream().getText()
+ }
+
+ def getProfile = { id ->
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst +
"/api/profile/text/?query_id=$id").openConnection()
+ conn.setRequestMethod("GET")
+ def encoding =
Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" :
context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ return conn.getInputStream().getText()
+ }
+
+ // Fetch profile text by token with retries for robustness
+ def getProfileWithToken = { token ->
+ String profileId = ""
+ int attempts = 0
+ while (attempts < 10 && (profileId == null || profileId == "")) {
+ List profileData = new
JsonSlurper().parseText(getProfileList()).data.rows
+ for (def profileItem in profileData) {
+ if (profileItem["Sql Statement"].toString().contains(token)) {
+ profileId = profileItem["Profile ID"].toString()
+ break
+ }
+ }
+ if (profileId == null || profileId == "") {
+ Thread.sleep(300)
+ }
+ attempts++
+ }
+ assertTrue(profileId != null && profileId != "", "Failed to find
profile for token: ${token}")
+ // ensure profile text is fully ready
+ Thread.sleep(800)
+ return getProfile(profileId).toString()
+ }
+
+ // Helper to extract RowsBloomFilterFiltered value from profile
+ def extractRowsBloomFilterFiltered = { String profileText ->
+ def lines = profileText.split("\n")
+ for (def line : lines) {
+ if (line.contains("RowsBloomFilterFiltered:")) {
+ def m = (line =~ /RowsBloomFilterFiltered:\s*([0-9]+)/)
+ if (m.find()) {
+ return m.group(1).toInteger()
+ }
+ }
+ }
+ return null
+ }
+
def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
for(int t = delta_time; t <= OpTimeout; t += delta_time){
alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
@@ -60,8 +118,6 @@ suite("test_ngram_bloomfilter_index_change") {
sql "set enable_profile=true"
sql "set profile_level=2"
- // Define test query
- def query = "SELECT /*+SET_VAR(enable_function_pushdown = true,
enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE
customer_name LIKE '%xxxx%' ORDER BY sale_id"
// Test Case 1: Test with enable_add_index_for_new_data = true
logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===")
// Create table
@@ -96,17 +152,12 @@ suite("test_ngram_bloomfilter_index_change") {
qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id"
// Test without NGRAM Bloom Filter index
- profile("sql_select_like_without_ngram_index_light_mode") {
- run {
- sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
- }
- }
+ def token1 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token1}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile1 = getProfileWithToken(token1)
+ def filtered1 = extractRowsBloomFilterFiltered(profile1)
+ logger.info("sql_select_like_without_ngram_index_light_mode:
RowsBloomFilterFiltered = ${filtered1}")
+ assertTrue(filtered1 != null && filtered1 == 0, "Expected
RowsBloomFilterFiltered = 0, but got ${filtered1}")
sql "set enable_add_index_for_new_data = true"
// Add NGRAM Bloom Filter index (should be immediate in light mode)
@@ -122,17 +173,12 @@ suite("test_ngram_bloomfilter_index_change") {
qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY
sale_id"
// Test with more data (should still filter correctly)
- profile("sql_select_like_with_ngram_index_light_mode_more_data") {
- run {
- sql "/* sql_select_like_with_ngram_index_light_mode_more_data */
${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
- }
- }
+ def token2 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token2}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile2 = getProfileWithToken(token2)
+ def filtered2 = extractRowsBloomFilterFiltered(profile2)
+ logger.info("sql_select_like_with_ngram_index_light_mode_more_data:
RowsBloomFilterFiltered = ${filtered2}")
+ assertTrue(filtered2 != null && filtered2 == 10, "Expected
RowsBloomFilterFiltered = 10, but got ${filtered2}")
// Drop index
sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
@@ -140,17 +186,12 @@ suite("test_ngram_bloomfilter_index_change") {
wait_for_last_build_index_finish(tableName, timeout)
// Test after dropping index
- profile("sql_select_like_with_ngram_index_light_mode_dropped") {
- run {
- sql "/* sql_select_like_with_ngram_index_light_mode_dropped */
${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
- }
- }
+ def token3 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token3}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile3 = getProfileWithToken(token3)
+ def filtered3 = extractRowsBloomFilterFiltered(profile3)
+ logger.info("sql_select_like_with_ngram_index_light_mode_dropped:
RowsBloomFilterFiltered = ${filtered3}")
+ assertTrue(filtered3 != null && filtered3 == 0, "Expected
RowsBloomFilterFiltered = 0, but got ${filtered3}")
// Test Case 2: Test with enable_add_index_for_new_data = false (schema
change mode)
logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===")
@@ -187,17 +228,12 @@ suite("test_ngram_bloomfilter_index_change") {
qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY
sale_id"
// Test without NGRAM Bloom Filter index
- profile("sql_select_like_without_ngram_index_schema_change_mode") {
- run {
- sql "/* sql_select_like_without_ngram_index_schema_change_mode */
${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
- }
- }
+ def token4 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token4}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile4 = getProfileWithToken(token4)
+ def filtered4 = extractRowsBloomFilterFiltered(profile4)
+ logger.info("sql_select_like_without_ngram_index_schema_change_mode:
RowsBloomFilterFiltered = ${filtered4}")
+ assertTrue(filtered4 != null && filtered4 == 0, "Expected
RowsBloomFilterFiltered = 0, but got ${filtered4}")
// Add NGRAM Bloom Filter index (will trigger schema change in this mode)
sql "ALTER TABLE ${tableName} ADD INDEX
idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' =
'1024', 'gram_size' = '3');"
@@ -205,17 +241,12 @@ suite("test_ngram_bloomfilter_index_change") {
wait_for_last_build_index_finish(tableName, timeout)
// Test after adding NGRAM Bloom Filter index (should filter existing data)
- profile("sql_select_like_with_ngram_index_schema_change_mode_added") {
- run {
- sql "/* sql_select_like_with_ngram_index_schema_change_mode_added
*/ ${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
- }
- }
+ def token5 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token5}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile5 = getProfileWithToken(token5)
+ def filtered5 = extractRowsBloomFilterFiltered(profile5)
+ logger.info("sql_select_like_with_ngram_index_schema_change_mode_added:
RowsBloomFilterFiltered = ${filtered5}")
+ assertTrue(filtered5 != null && filtered5 == 10, "Expected
RowsBloomFilterFiltered = 10, but got ${filtered5}")
// Insert more data after index is built
insertTestData()
@@ -223,17 +254,12 @@ suite("test_ngram_bloomfilter_index_change") {
qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER
BY sale_id"
// Test with more data (should filter all data)
- profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") {
- run {
- sql "/*
sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
- }
- }
+ def token6 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token6}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile6 = getProfileWithToken(token6)
+ def filtered6 = extractRowsBloomFilterFiltered(profile6)
+
logger.info("sql_select_like_with_ngram_index_schema_change_mode_more_data:
RowsBloomFilterFiltered = ${filtered6}")
+ assertTrue(filtered6 != null && filtered6 == 20, "Expected
RowsBloomFilterFiltered = 20, but got ${filtered6}")
// Drop index
sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
@@ -241,17 +267,12 @@ suite("test_ngram_bloomfilter_index_change") {
wait_for_last_build_index_finish(tableName, timeout)
// Test after dropping index
- profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") {
- run {
- sql "/*
sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
- }
- }
+ def token7 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token7}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile7 = getProfileWithToken(token7)
+ def filtered7 = extractRowsBloomFilterFiltered(profile7)
+ logger.info("sql_select_like_with_ngram_index_schema_change_mode_dropped:
RowsBloomFilterFiltered = ${filtered7}")
+ assertTrue(filtered7 != null && filtered7 == 0, "Expected
RowsBloomFilterFiltered = 0, but got ${filtered7}")
// Test Case 3: Test different scenarios for index lifecycle
logger.info("=== Test Case 3: Index lifecycle with
enable_add_index_for_new_data = true ===")
@@ -294,17 +315,12 @@ suite("test_ngram_bloomfilter_index_change") {
qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY
sale_id"
// Test filtering with index added before data insertion
- profile("sql_select_like_with_ngram_index_lifecycle_test") {
- run {
- sql "/* sql_select_like_with_ngram_index_lifecycle_test */
${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
- }
- }
+ def token8 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token8}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile8 = getProfileWithToken(token8)
+ def filtered8 = extractRowsBloomFilterFiltered(profile8)
+ logger.info("sql_select_like_with_ngram_index_lifecycle_test:
RowsBloomFilterFiltered = ${filtered8}")
+ assertTrue(filtered8 != null && filtered8 == 10, "Expected
RowsBloomFilterFiltered = 10, but got ${filtered8}")
// Insert more data
insertTestData()
@@ -312,17 +328,12 @@ suite("test_ngram_bloomfilter_index_change") {
qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id"
// Test filtering with more data
- profile("sql_select_like_with_ngram_index_lifecycle_final") {
- run {
- sql "/* sql_select_like_with_ngram_index_lifecycle_final */
${query}"
- sleep(1000)
- }
-
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
- }
- }
+ def token9 = UUID.randomUUID().toString()
+ sql "SELECT /*+SET_VAR(enable_function_pushdown = true)*/ '${token9}', *
FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ def profile9 = getProfileWithToken(token9)
+ def filtered9 = extractRowsBloomFilterFiltered(profile9)
+ logger.info("sql_select_like_with_ngram_index_lifecycle_final:
RowsBloomFilterFiltered = ${filtered9}")
+ assertTrue(filtered9 != null && filtered9 == 20, "Expected
RowsBloomFilterFiltered = 20, but got ${filtered9}")
// Final cleanup
sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
