airborne12 commented on code in PR #67180: URL: https://github.com/apache/doris/pull/67180#discussion_r4090358563
########## be/test/storage/index/snii/bench/phrase_candidate_pushdown_bench_test.cpp: ########## @@ -0,0 +1,353 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Candidate pushdown into multi-term phrase queries, measured for both inverted index storage +// formats. One synthetic log corpus is indexed through the production column writer as CLucene (V2) +// and as SNII. Every phrase then runs through the production reader with and without +// IndexQueryContext::candidate_rows, and every restricted result is checked against the full result +// intersected with the candidates. +// +// The test is DISABLED_ so CI never runs it; it is still compiled into doris_be_test. Use a RELEASE +// UT build (BUILD_TYPE_UT=RELEASE in custom_env.sh) for representative numbers: +// +// GTEST_ALSO_RUN_DISABLED_TESTS=1 ./run-be-ut.sh --run \ +// --filter='*PhraseCandidatePushdownBench*' -j <N> +// +// PHRASE_CANDIDATE_BENCH_DOCS sets the segment size (default 200000) and +// PHRASE_CANDIDATE_BENCH_ITERATIONS the samples per measurement (default 10). Times are medians of +// per-query thread CPU time, which moves far less than wall time on a shared machine. + +#include <fmt/format.h> +#include <gen_cpp/PaloInternalService_types.h> +#include <gtest/gtest.h> + +#include <algorithm> +#include <cstdint> +#include <cstdlib> +#include <ctime> +#include <iomanip> +#include <iostream> +#include <memory> +#include <random> +#include <string> +#include <string_view> +#include <vector> + +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_query_context.h" +#include "storage/index/index_writer.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_index_reader.h" +#include "storage/olap_common.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +constexpr const char* kBenchDir = "./ut_dir/phrase_candidate_pushdown_bench"; +constexpr const char* kColumnName = "1"; +constexpr double kCandidateRatios[] = {0.001, 0.01, 0.05, 0.1, 0.224, 0.3, 0.5}; + +struct BenchQuery { + const char* label; + InvertedIndexQueryType type; + const char* text; +}; + +// Frequent two- and four-term phrases, a frequent lead with an expanding numeric tail, a +// log-search shape with a long lead, and a rare phrase whose result is far below any candidate set. +constexpr BenchQuery kQueries[] = {{.label = "exact_2", + .type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .text = "retry attempt"}, + {.label = "exact_4", + .type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .text = "retry attempt 2 job"}, + {.label = "prefix_2", + .type = InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, + .text = "order 12"}, + {.label = "prefix_5", + .type = InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, + .text = "retry attempt 1 job 88"}, + {.label = "rare_exact", + .type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .text = "request 424242 completed"}}; Review Comment: Confirmed the modulo mismatch. Commit `93ffe53684e` seeds `Request 424242 completed latency 42` at `doc_count / 4` and asserts that the unrestricted result is nonempty. I ran the disabled benchmark under ASAN with 4,000 documents and one iteration: it passed 1/1. Both V2 and SNII returned one match for the clustered 0.1% candidate range, confirming that the rare phrase is indexed and the restricted path is exercised. The short run validates path coverage, not performance numbers. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
