This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 053f4511753 branch-4.0: [fix](profile) ensure file cache profile stats
output in cloud mode #57464 (#57534)
053f4511753 is described below
commit 053f4511753b37246e646637ef2d7aa8aaf0c39c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Oct 31 09:23:12 2025 +0800
branch-4.0: [fix](profile) ensure file cache profile stats output in cloud
mode #57464 (#57534)
Cherry-picked from #57464
Co-authored-by: zzzxl <[email protected]>
---
be/src/vec/exec/scan/olap_scanner.cpp | 5 +-
.../test_inverted_index_io_timer.groovy | 141 +++++++++++++++++++++
2 files changed, 143 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp
b/be/src/vec/exec/scan/olap_scanner.cpp
index afb9b546c49..61d099c971c 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -769,9 +769,8 @@ void OlapScanner::_collect_profile_before_close() {
inverted_index_profile.update(local_state->_index_filter_profile.get(),
&stats.inverted_index_stats);
- // only cloud deploy mode will use file cache. and keep the same with
FileScanner
- if (config::is_cloud_mode() && config::enable_file_cache &&
- _state->query_options().enable_file_cache) {
+ // only cloud deploy mode will use file cache.
+ if (config::is_cloud_mode() && config::enable_file_cache) {
io::FileCacheProfileReporter
cache_profile(local_state->_segment_profile.get());
cache_profile.update(&stats.file_cache_stats);
}
diff --git
a/regression-test/suites/inverted_index_p0/test_inverted_index_io_timer.groovy
b/regression-test/suites/inverted_index_p0/test_inverted_index_io_timer.groovy
new file mode 100644
index 00000000000..54f9c25f8a0
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_inverted_index_io_timer.groovy
@@ -0,0 +1,141 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.util.regex.Pattern
+
+suite('test_inverted_index_io_timer', 'p0') {
+ if (!isCloudMode()) {
+ return;
+ }
+
+ def indexTbName1 = "test_inverted_index_io_timer_tbl"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+
+ // Create table with inverted index using httplogs schema
+ sql """
+ CREATE TABLE ${indexTbName1} (
+ `@timestamp` int(11) NULL COMMENT "",
+ `clientip` varchar(20) NULL COMMENT "",
+ `request` text NULL COMMENT "",
+ `status` int(11) NULL COMMENT "",
+ `size` int(11) NULL COMMENT "",
+ INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
+ INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" =
"english", "support_phrase" = "true") COMMENT '',
+ INDEX status_idx (`status`) USING INVERTED COMMENT '',
+ INDEX size_idx (`size`) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`@timestamp`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ // Define data loading function
+ def load_httplogs_data = {table_name, label, read_flag, format_flag,
file_name, ignore_failure=false,
+ expected_succ_rows = -1, load_to_single_tablet =
'true' ->
+
+ // load the json data
+ streamLoad {
+ table "${table_name}"
+
+ // set http request header params
+ set 'label', label + "_" + UUID.randomUUID().toString()
+ set 'read_json_by_line', read_flag
+ set 'format', format_flag
+ file file_name // import json file
+ time 10000 // limit inflight 10s
+ if (expected_succ_rows >= 0) {
+ set 'max_filter_ratio', '1'
+ }
+
+ // if declared a check callback, the default check condition will
ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (ignore_failure && expected_succ_rows < 0) { return }
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ if (expected_succ_rows >= 0) {
+ assertEquals(json.NumberLoadedRows, expected_succ_rows)
+ } else {
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows +
json.NumberUnselectedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+ }
+
+ try {
+ // Load 1000 documents
+ load_httplogs_data.call(indexTbName1, indexTbName1, 'true', 'json',
'documents-1000.json')
+
+ sql "sync"
+
+ // Enable profile
+ sql """ set enable_profile = true; """
+ sql """ set profile_level = 2; """
+ sql """ set enable_sql_cache = false; """
+ sql """ set enable_inverted_index_searcher_cache = false; """
+ sql """ set enable_inverted_index_query_cache = false; """
+ sql """ set enable_common_expr_pushdown = true; """
+ sql """ set enable_common_expr_pushdown_for_inverted_index = true; """
+ sql """ set enable_match_without_inverted_index = false; """
+
+ // Execute query with inverted index using profile
+ def queryId =
"test_inverted_index_io_timer_${System.currentTimeMillis()}"
+ profile("${queryId}") {
+ run {
+ sql "/* ${queryId} */ select * from ${indexTbName1} where
request match 'images' order by `@timestamp` limit 10"
+ }
+
+ check { profileString, exception ->
+ def local = 0
+ def remote = 0
+
+ def localMatcher =
Pattern.compile("InvertedIndexNumLocalIOTotal:\\s*(\\d+)").matcher(profileString)
+ if (localMatcher.find()) {
+ local = Integer.parseInt(localMatcher.group(1))
+ log.info("InvertedIndexNumLocalIOTotal: {}", local)
+ }
+
+ def remoteMatcher =
Pattern.compile("InvertedIndexNumRemoteIOTotal:\\s*(\\d+)").matcher(profileString)
+ if (remoteMatcher.find()) {
+ remote = Integer.parseInt(remoteMatcher.group(1))
+ log.info("InvertedIndexNumRemoteIOTotal: {}", remote)
+ }
+
+ def total = local + remote
+ assertTrue(total > 0, "InvertedIndexNumLocalIOTotal +
InvertedIndexNumRemoteIOTotal should be > 0, got: ${total} (local=${local},
remote=${remote})")
+ }
+ }
+
+ // Also verify the query returns correct result
+ def result = sql "select count(*) from ${indexTbName1} where request
match 'images'"
+ assertTrue(result[0][0] > 0, "Should have at least one row matching
'images'")
+
+ log.info("Test completed successfully: InvertedIndexIOTimer is greater
than 0")
+ } finally {
+ // Clean up
+ // sql "DROP TABLE IF EXISTS ${indexTbName1}"
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
