Copilot commented on code in PR #61746: URL: https://github.com/apache/doris/pull/61746#discussion_r2992430447
########## regression-test/suites/variant_p0/test_variant_multi_index_schema_change.groovy: ########## @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_multi_index_schema_change", "p0,nonConcurrent") { + if (!isCloudMode()) { + logger.info("not cloud mode, skip this test") + return + } + + sql """ set enable_match_without_inverted_index = false """ + sql """ set enable_common_expr_pushdown = true """ + sql """ set inverted_index_skip_threshold = 0 """ + sql """ set default_variant_enable_typed_paths_to_sparse = false """ + + def createVariantTable = { tableName -> + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `id` bigint NULL, + `tag` string NULL, + `var` variant<properties ("variant_max_subcolumns_count" = "10")> NOT NULL, + INDEX idx_a_d (`var`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX idx_a_d_2 (`var`) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ) + """ + } + + def insertBatch = { tableName -> + sql """ + INSERT INTO ${tableName} VALUES + (1, 'hello', '{"string" : "hello", "array_string" : ["hello"]}'), + (2, 'world', '{"string" : "world", "array_string" : ["world"]}'), + (3, 'hello', '{"string" : "hello", "array_string" : ["hello"]}'), + (4, 'world', '{"string" : "world", "array_string" : ["world"]}'), + (5, 'hello', '{"string" : "hello", "array_string" : ["hello"]}') + """ + } + + def loadTable = { tableName -> + insertBatch(tableName) + for (int i = 0; i < 10; i++) { + insertBatch(tableName) + } + trigger_and_wait_compaction(tableName, "cumulative") + } + + def waitSchemaChange = { tableName -> + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + time 600 + } + } + + def assertPhraseCount = { tableName -> + assertEquals(33, sql(""" SELECT COUNT() FROM ${tableName} WHERE var['string'] MATCH_PHRASE 'hello' """)[0][0]) + } + + def assertNestedIndexFileReady = { tableName -> + def backendIdToBackendIP = [:] + def backendIdToBackendHttpPort = [:] + getBackendIpHttpPort(backendIdToBackendIP, backendIdToBackendHttpPort) + def tablets = sql_return_maparray(""" SHOW TABLETS FROM ${tableName} """) + String tabletId = tablets[0].TabletId + String backendId = tablets[0].BackendId + String ip = backendIdToBackendIP.get(backendId) + String port = backendIdToBackendHttpPort.get(backendId) + check_nested_index_file(ip, port, tabletId, 2, 3, "V2") Review Comment: `check_nested_index_file(...)` hard-codes index storage format as "V2", but the table DDL does not set `"inverted_index_storage_format"` in PROPERTIES. This makes the assertion dependent on cluster defaults (and could flip to V3/V1), causing test flakiness. Consider setting `"inverted_index_storage_format" = "V2"` explicitly in the CREATE TABLE PROPERTIES (or loosening the assertion to accept the actual format returned by the API). ```suggestion check_nested_index_file(ip, port, tabletId, 2, 3, null) ``` ########## regression-test/suites/variant_p0/test_variant_multi_index_schema_change.groovy: ########## @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_multi_index_schema_change", "p0,nonConcurrent") { + if (!isCloudMode()) { + logger.info("not cloud mode, skip this test") + return + } + + sql """ set enable_match_without_inverted_index = false """ + sql """ set enable_common_expr_pushdown = true """ + sql """ set inverted_index_skip_threshold = 0 """ + sql """ set default_variant_enable_typed_paths_to_sparse = false """ + + def createVariantTable = { tableName -> + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `id` bigint NULL, + `tag` string NULL, + `var` variant<properties ("variant_max_subcolumns_count" = "10")> NOT NULL, + INDEX idx_a_d (`var`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX idx_a_d_2 (`var`) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ) + """ + } + + def insertBatch = { tableName -> + sql """ + INSERT INTO ${tableName} VALUES + (1, 'hello', '{"string" : "hello", "array_string" : ["hello"]}'), + (2, 'world', '{"string" : "world", "array_string" : ["world"]}'), + (3, 'hello', '{"string" : "hello", "array_string" : ["hello"]}'), + (4, 'world', '{"string" : "world", "array_string" : ["world"]}'), + (5, 'hello', '{"string" : "hello", "array_string" : ["hello"]}') + """ + } + + def loadTable = { tableName -> + insertBatch(tableName) + for (int i = 0; i < 10; i++) { + insertBatch(tableName) + } + trigger_and_wait_compaction(tableName, "cumulative") + } + + def waitSchemaChange = { tableName -> + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + time 600 + } + } + + def assertPhraseCount = { tableName -> + assertEquals(33, sql(""" SELECT COUNT() FROM ${tableName} WHERE var['string'] MATCH_PHRASE 'hello' """)[0][0]) + } + + def assertNestedIndexFileReady = { tableName -> + def backendIdToBackendIP = [:] + def backendIdToBackendHttpPort = [:] + getBackendIpHttpPort(backendIdToBackendIP, backendIdToBackendHttpPort) + def tablets = sql_return_maparray(""" SHOW TABLETS FROM ${tableName} """) + String tabletId = tablets[0].TabletId + String backendId = tablets[0].BackendId + String ip = backendIdToBackendIP.get(backendId) + String port = backendIdToBackendHttpPort.get(backendId) + check_nested_index_file(ip, port, tabletId, 2, 3, "V2") Review Comment: `check_nested_index_file(ip, port, tabletId, 2, 3, ...)` assumes there will be exactly 2 rowsets after `trigger_and_wait_compaction(tableName, "cumulative")`. Rowset count after compaction is environment/strategy dependent, and other regression tests compute `activeRowsetCount` from `be_show_tablet_status` rather than hard-coding it. To avoid flaky failures, consider deriving the expected rowset count dynamically (or asserting a range / invariant like "< before"), then pass that value into `check_nested_index_file`. ```suggestion // Derive the active rowset count dynamically to avoid hard-coded assumptions def tabletStatusList = be_show_tablet_status(tableName) int activeRowsetCount = 0 for (status in tabletStatusList) { if (status.TabletId == tabletId) { // Use the rowset count reported by the backend for this tablet activeRowsetCount = (status.RowsetCount as int) break } } assertTrue(activeRowsetCount > 0) check_nested_index_file(ip, port, tabletId, activeRowsetCount, 3, "V2") ``` ########## regression-test/suites/variant_p0/test_variant_multi_index_schema_change.groovy: ########## @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_multi_index_schema_change", "p0,nonConcurrent") { + if (!isCloudMode()) { + logger.info("not cloud mode, skip this test") + return + } + + sql """ set enable_match_without_inverted_index = false """ + sql """ set enable_common_expr_pushdown = true """ + sql """ set inverted_index_skip_threshold = 0 """ + sql """ set default_variant_enable_typed_paths_to_sparse = false """ + + def createVariantTable = { tableName -> + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `id` bigint NULL, + `tag` string NULL, + `var` variant<properties ("variant_max_subcolumns_count" = "10")> NOT NULL, + INDEX idx_a_d (`var`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX idx_a_d_2 (`var`) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ) + """ + } + + def insertBatch = { tableName -> + sql """ + INSERT INTO ${tableName} VALUES + (1, 'hello', '{"string" : "hello", "array_string" : ["hello"]}'), + (2, 'world', '{"string" : "world", "array_string" : ["world"]}'), + (3, 'hello', '{"string" : "hello", "array_string" : ["hello"]}'), + (4, 'world', '{"string" : "world", "array_string" : ["world"]}'), + (5, 'hello', '{"string" : "hello", "array_string" : ["hello"]}') + """ + } + + def loadTable = { tableName -> + insertBatch(tableName) + for (int i = 0; i < 10; i++) { + insertBatch(tableName) + } + trigger_and_wait_compaction(tableName, "cumulative") + } + + def waitSchemaChange = { tableName -> + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + time 600 + } + } + + def assertPhraseCount = { tableName -> + assertEquals(33, sql(""" SELECT COUNT() FROM ${tableName} WHERE var['string'] MATCH_PHRASE 'hello' """)[0][0]) + } + + def assertNestedIndexFileReady = { tableName -> + def backendIdToBackendIP = [:] + def backendIdToBackendHttpPort = [:] + getBackendIpHttpPort(backendIdToBackendIP, backendIdToBackendHttpPort) + def tablets = sql_return_maparray(""" SHOW TABLETS FROM ${tableName} """) + String tabletId = tablets[0].TabletId + String backendId = tablets[0].BackendId + String ip = backendIdToBackendIP.get(backendId) + String port = backendIdToBackendHttpPort.get(backendId) + check_nested_index_file(ip, port, tabletId, 2, 3, "V2") Review Comment: The expected per-segment `indices_count` is set to `3`, but the table defines two inverted indexes on `var` and the inserted JSON has two extracted paths (`string`, `array_string`). A very similar variant multi-index test asserts `indices_count = 4` for this shape (2 indexes × 2 paths). Unless there is a documented reason this schema change produces only 3 indices, this expectation is likely incorrect and will fail. Consider aligning the expected `indices_count` with the actual number of built indices (e.g., 4) or deriving it from `SHOW INDEX`/returned JSON instead of hard-coding. ```suggestion check_nested_index_file(ip, port, tabletId, 2, 4, "V2") ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
