englefly commented on code in PR #69322: URL: https://github.com/apache/doris/pull/69322#discussion_r4385705538
########## regression-test/suites/query_p0/hbo/hbo_row_count_drift_test.groovy: ########## @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_row_count_drift_test", "nonConcurrent") { + // A hbo key identifies a plan pattern, not a data state: the visible version and the scanned + // rows of every scan are recorded as a baseline inside the struct info (and stripped before the + // fingerprint is hashed), so an injected entry keeps matching while its tables grow. The read + // side decides whether the recorded row count may still be applied by comparing that baseline + // with the data state of the query which wants to use it: + // - the recorded version is still the current one -> live (applied) + // - the data moved by at most hbo_row_count_change_ratio (10%) -> drifted (applied) + // - it moved further than that -> stale (skipped) + // - the entry records no state at all -> unknown (applied) + // The recorded row count is never scaled: the entry either applies with the value the user + // measured or does not apply at all. The comparison uses the rows of the SELECTED partitions, + // so a table which grows in partitions a query does not read does not invalidate its entries - + // while HBO SHOW STATISTICS / HBO DELETE STALE STATISTICS, which cannot run the query of an + // entry, compare the recorded state with the catalog as a whole and report such an entry as + // unknown. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + + sql "drop table if exists hbo_dr_t;" + sql "drop table if exists hbo_dr_p;" + sql """create table hbo_dr_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_dr_t select number, number % 100 from numbers("number" = "1000");""" + sql """analyze table hbo_dr_t with sync;""" + sql """create table hbo_dr_p(a int, b int, dt int) partition by range(dt) ( + partition p1 values less than ('100'), + partition p2 values less than ('200')) + distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_dr_p select number, number % 100, 10 from numbers("number" = "1000");""" + sql """insert into hbo_dr_p select number, number % 100, 110 from numbers("number" = "1000");""" + sql """analyze table hbo_dr_p with sync;""" + + def prevInfoCollection = (sql "show global variables like 'enable_hbo_info_collection'")[0][1].toString() + def prevRatio = (sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'hbo_row_count_change_ratio'; """)[0][1].toString() + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + def injected = [] + try { + sql "set global enable_hbo_info_collection=true;" + def explainText = { String q -> (sql """ explain $q """).flatten().join("\n") } + // the annotation line of one entry (the read side reports the data state verdict on it) + def entryLineOf = { String text, String fingerprint -> + def line = text.split("\n").find { it.contains("fingerprint='${fingerprint}'") } + assertTrue(line != null, "no annotation line for ${fingerprint} in:\n" + text) + line + } + def verdictOf = { String text, String fingerprint -> + def matcher = (entryLineOf(text, fingerprint) =~ /(used=\S+|skipped=\S+)/) + matcher.find() ? matcher.group(1) : "" + } + def stateOf = { String like -> + def rows = sql """ HBO SHOW PINNED STATISTICS LIKE '${like}'; """ + assertEquals(1, rows.size(), rows.toString()) + rows[0][7].toString() + } + + // ---------------------------------------------------------------- one table, no pruning + def filterQuery = "select * from hbo_dr_t where b = 1" + def matcher = (explainText(filterQuery) =~ + /filter-on-scan\(table=[^)]*hbo_dr_t[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(matcher.find(), "no filter annotation found:\n" + explainText(filterQuery)) + def fingerprint = matcher.group(1) + def structCanonical = matcher.group(2) + // the struct info records the data state the entry is measured in; the row count is tagged + // 'r' (measured from the partitions) or 'e' (derived from the table), which the assertions + // below normalise because both are the same number for these tables + // the row-count tag is normalized (a measurement and an estimate are compared as one), and the + // identity a scan token carries - the id of its table and the alias of its relation - is + // normalized away: these assertions check the partitions and the row state of a scan, while the + // identity itself is pinned by the unit tests and the golden corpus + def normalizeRows = { String text -> + text.replaceAll(/,[re](\d+)/, ',r$1').replaceAll(/#t[ia]:[^,}]*/, '') + } + assertTrue(normalizeRows(structCanonical).contains("S{internal.hbo_test.hbo_dr_t,v2,r1000}"), + structCanonical) + // ... and the row count carries the tag of how it was obtained: 'r' when every selected + // partition reported its own count, 'e' when it was derived from the table + assertTrue((normalizeRows(structCanonical) =~ /S\{internal\.hbo_test\.hbo_dr_t,v\d+,[re]\d+\}/).find(), + structCanonical) + sql """ HBO SET STATISTICS VALUE=600 LITERAL_MODE=WITH_LITERAL FINGERPRINT='${fingerprint}' + STRUCT='${structCanonical}'; """ + injected.add(fingerprint) + + // (1) nothing changed: the entry applies with the recorded data state + assertEquals("used=live", verdictOf(explainText(filterQuery), fingerprint)) + assertEquals("live", stateOf('%hbo\\_dr\\_t%')) + + // (2) 1% more rows: the key of the entry did not change (a growing table does not invalidate + // an entry any more), the read side applies it and reports how far the data drifted + sql """ insert into hbo_dr_t select number + 1000, number % 100 from numbers("number" = "10"); """ + assertEquals("used=drifted(rows=1010,rec=1000,+1.0%)", verdictOf(explainText(filterQuery), fingerprint)) Review Comment: 行数上报如果不及时可能会有不稳定情况. 如果实际流水线上出现不稳定再删掉这个case -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
