This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 7974b035493 branch-4.2: [improvement](be) Keep hot SNII term lookups
across queries #69879 (#69959)
7974b035493 is described below
commit 7974b0354932f2c460f69acf6891427343466975
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Nov 3 10:09:25 2026 +0800
branch-4.2: [improvement](be) Keep hot SNII term lookups across queries
#69879 (#69959)
Cherry-picked from #69879
Co-authored-by: Jack <[email protected]>
---
.../inverted/snii/reader/logical_index_reader.cpp | 125 ++++++++++-
.../inverted/snii/reader/logical_index_reader.h | 18 +-
.../index/inverted/snii/snii_index_reader.cpp | 3 +
.../index/inverted/snii/snii_query_test.cpp | 238 +++++++++++++++++++++
4 files changed, 372 insertions(+), 12 deletions(-)
diff --git a/be/src/storage/index/inverted/snii/reader/logical_index_reader.cpp
b/be/src/storage/index/inverted/snii/reader/logical_index_reader.cpp
index 186dd7b936d..3e58ac1e864 100644
--- a/be/src/storage/index/inverted/snii/reader/logical_index_reader.cpp
+++ b/be/src/storage/index/inverted/snii/reader/logical_index_reader.cpp
@@ -21,8 +21,12 @@
#include <cstdlib>
#include <future>
#include <limits>
+#include <list>
#include <memory>
#include <mutex>
+#include <string>
+#include <string_view>
+#include <unordered_map>
#include <utility>
#include <vector>
@@ -53,6 +57,15 @@ struct LogicalIndexReader::NormsCacheState {
std::shared_future<Status> in_flight;
};
+// Exact lookup results of an on-demand dictionary, least recently used last.
+struct LogicalIndexReader::TermLookupCache {
+ using Kept = std::pair<std::string, BatchLookupResult>;
+
+ std::mutex mutex;
+ std::list<Kept> order;
+ std::unordered_map<std::string_view, std::list<Kept>::iterator> index;
+};
+
using format::BlockRef;
using format::bsbf_hash;
using format::DictBlockDirectoryReader;
@@ -73,6 +86,17 @@ constexpr uint64_t kMaxDictLookupBatchBytes = 4ULL * 1024 *
1024;
// for the state/data/vector allocations. The framed norms bytes are charged
// separately at their exact validated length.
constexpr size_t kNormsAllocationOverheadCharge = 128;
+// The term lookup cache keeps at most this many terms per reader, each at
most this long and with
+// at most this many bytes of inline postings, so its footprint has a fixed
bound.
+constexpr size_t kTermLookupCacheTerms = 32;
+constexpr size_t kTermLookupMaxTermBytes = 64;
+constexpr size_t kTermLookupMaxInlineBytes = 256;
+// One kept term at those bounds: its list node (links and the key/result
pair), its hash node and
+// bucket, the heap of the key and of the entry's term, the inline postings,
and allocator
+// bookkeeping for each of those allocations.
+constexpr size_t kTermLookupEntryCharge =
+ sizeof(std::pair<std::string, LogicalIndexReader::BatchLookupResult>)
+ 2 * sizeof(void*) +
+ 64 + 2 * (kTermLookupMaxTermBytes + 1) + kTermLookupMaxInlineBytes + 6
* 16;
// L0/L1 tiering threshold (bytes). Defaults to kBsbfResidentMaxBytes; the env
// SNII_BSBF_RESIDENT_MAX overrides it for tuning and for exercising the
@@ -488,6 +512,10 @@ size_t LogicalIndexReader::memory_usage() const {
for (const auto& block : resident_dict_blocks_) {
bytes += sizeof(block) + block.bytes.capacity() +
block.reader.heap_bytes();
}
+ // The term lookup cache fills lazily as well, so its bounded footprint is
reserved in full.
+ if (term_lookups_ != nullptr) {
+ bytes += sizeof(TermLookupCache) + kTermLookupCacheTerms *
kTermLookupEntryCharge;
+ }
// Norms are loaded lazily, but the searcher-cache charge is fixed when
this
// reader is inserted. Reserve their complete framed size and heap state up
// front so later allocation is already accounted for. Saturation prevents
@@ -508,6 +536,12 @@ size_t LogicalIndexReader::memory_usage() const {
return bytes;
}
+void LogicalIndexReader::enable_term_lookup_cache() {
+ if (resident_dict_blocks_.empty() && dbd_.n_blocks() != 0) {
+ term_lookups_ = std::make_shared<TermLookupCache>();
+ }
+}
+
Status LogicalIndexReader::open_norms(format::NormsPodReader* out) const {
if (out == nullptr) {
return Status::Error<ErrorCode::INVALID_ARGUMENT, false>(
@@ -692,6 +726,18 @@ Status LogicalIndexReader::lookup(std::string_view term,
bool* found, DictEntry*
if (!maybe) {
return Status::OK();
}
+ if (term_lookups_ != nullptr) {
+ BatchLookupResult kept;
+ if (find_kept_lookup(term, &kept)) {
+ *found = kept.found;
+ if (kept.found) {
+ *entry = std::move(kept.entry);
+ *frq_base = kept.frq_base;
+ *prx_base = kept.prx_base;
+ }
+ return Status::OK();
+ }
+ }
// Use a resident small-DICT block when present; otherwise read the DICT
// block on demand and parse it with the same validation path used at open.
@@ -702,16 +748,68 @@ Status LogicalIndexReader::lookup(std::string_view term,
bool* found, DictEntry*
bool hit = false;
RETURN_IF_ERROR(br->find_term(term, &hit, entry));
- if (!hit) {
- return Status::OK();
+ if (hit) {
+ *found = true;
+ *frq_base = br->frq_base();
+ *prx_base = br->prx_base();
}
-
- *found = true;
- *frq_base = br->frq_base();
- *prx_base = br->prx_base();
+ keep_lookup(term, hit, *entry, br->frq_base(), br->prx_base());
return Status::OK();
}
+bool LogicalIndexReader::find_kept_lookup(std::string_view term,
BatchLookupResult* out) const {
+ TermLookupCache& cache = *term_lookups_;
+ std::lock_guard lock(cache.mutex);
+ const auto it = cache.index.find(term);
+ if (it == cache.index.end()) {
+ return false;
+ }
+ cache.order.splice(cache.order.begin(), cache.order, it->second);
+ *out = it->second->second;
+ return true;
+}
+
+void LogicalIndexReader::keep_lookup(std::string_view term, bool found, const
DictEntry& entry,
+ uint64_t frq_base, uint64_t prx_base)
const {
+ // Within these bounds the charge memory_usage() reserved covers the cache
at its fullest.
+ if (term_lookups_ == nullptr || term.size() > kTermLookupMaxTermBytes ||
+ (found && (entry.term.size() > kTermLookupMaxTermBytes ||
+ entry.frq_bytes.size() + entry.prx_bytes.size() >
kTermLookupMaxInlineBytes))) {
+ return;
+ }
+ BatchLookupResult result;
+ if (found) {
+ result.found = true;
+ result.entry = entry;
+ result.frq_base = frq_base;
+ result.prx_base = prx_base;
+ }
+ TermLookupCache& cache = *term_lookups_;
+ std::lock_guard lock(cache.mutex);
+ if (cache.index.contains(term)) {
+ return;
+ }
+ cache.order.emplace_front(std::string(term), std::move(result));
+ cache.index.emplace(cache.order.front().first, cache.order.begin());
+ if (cache.order.size() > kTermLookupCacheTerms) {
+ cache.index.erase(cache.order.back().first);
+ cache.order.pop_back();
+ }
+}
+
+void LogicalIndexReader::keep_group_lookups(const BatchLookupState& state,
+ const BatchLookupGroup& group)
const {
+ if (term_lookups_ == nullptr) {
+ return;
+ }
+ for (size_t i = group.begin; i < group.end; ++i) {
+ const size_t term_index = state.candidates_[i].term_index;
+ const BatchLookupResult& result = (*state.results_)[term_index];
+ keep_lookup((*state.terms_)[term_index], result.found, result.entry,
result.frq_base,
+ result.prx_base);
+ }
+}
+
Status LogicalIndexReader::may_contain(std::string_view term, bool*
maybe_present) const {
uint32_t ordinal = 0;
return locate_candidate_dict_block(term, maybe_present, &ordinal);
@@ -740,14 +838,15 @@ Status
LogicalIndexReader::locate_candidate_dict_block(std::string_view term, bo
Status LogicalIndexReader::collect_batch_lookup_groups(
const std::vector<std::string>& terms,
std::vector<BatchLookupCandidate>* candidates,
- std::vector<BatchLookupGroup>* groups) const {
+ std::vector<BatchLookupGroup>* groups, std::vector<BatchLookupResult>*
kept_results) const {
candidates->clear();
candidates->reserve(terms.size());
for (size_t i = 0; i < terms.size(); ++i) {
bool maybe = false;
uint32_t ordinal = 0;
RETURN_IF_ERROR(locate_candidate_dict_block(terms[i], &maybe,
&ordinal));
- if (maybe) {
+ if (maybe &&
+ (kept_results == nullptr || !find_kept_lookup(terms[i],
&(*kept_results)[i]))) {
candidates->push_back({i, ordinal});
}
}
@@ -792,7 +891,8 @@ Status LogicalIndexReader::prepare_lookup_batch(const
std::vector<std::string>&
return lookup(terms.front(), &result.found, &result.entry,
&result.frq_base,
&result.prx_base);
}
- RETURN_IF_ERROR(collect_batch_lookup_groups(terms, &state->candidates_,
&state->groups_));
+ RETURN_IF_ERROR(collect_batch_lookup_groups(terms, &state->candidates_,
&state->groups_,
+ term_lookups_ != nullptr ?
results : nullptr));
if (!resident_dict_blocks_.empty()) {
for (const BatchLookupGroup& group : state->groups_) {
const DictBlockReader* block_reader = nullptr;
@@ -893,6 +993,7 @@ Status
LogicalIndexReader::consume_lookup_wave(BatchLookupState* state,
RETURN_IF_ERROR(resolve_batch_lookup_entries(*state->terms_,
state->candidates_,
state->groups_[block.group_index],
block_reader,
state->results_));
+ keep_group_lookups(*state, state->groups_[block.group_index]);
}
if (!state->pending_.empty()) {
state->next_group_ = state->pending_.back().group_index + 1;
@@ -916,8 +1017,10 @@ Status LogicalIndexReader::lookup_batch(const
std::vector<std::string>& terms,
std::shared_ptr<const DecodedDictBlock> pin;
RETURN_IF_ERROR(dict_block_reader_for_ordinal(group.ordinal,
/*cache=*/nullptr, &pin,
&block_reader));
- return resolve_batch_lookup_entries(terms, state.candidates_, group,
*block_reader,
- results);
+ RETURN_IF_ERROR(resolve_batch_lookup_entries(terms, state.candidates_,
group, *block_reader,
+ results));
+ keep_group_lookups(state, group);
+ return Status::OK();
}
while (!state.done()) {
io::BatchRangeFetcher fetcher(reader_, /*coalesce_gap=*/0);
diff --git a/be/src/storage/index/inverted/snii/reader/logical_index_reader.h
b/be/src/storage/index/inverted/snii/reader/logical_index_reader.h
index bb7a99a6d2a..2e707f9dab8 100644
--- a/be/src/storage/index/inverted/snii/reader/logical_index_reader.h
+++ b/be/src/storage/index/inverted/snii/reader/logical_index_reader.h
@@ -279,8 +279,15 @@ public:
Status null_docids_scan_memory(NullDocidsScanMemory* out) const;
size_t memory_usage() const;
+ // Keeps the results of exact lookups in an on-demand dictionary for the
queries that share
+ // this reader, so a hot term's next lookup skips the read, zstd and CRC
of its DICT block.
+ // The cache is bounded and memory_usage() reserves its whole footprint,
so enable it before
+ // the reader is charged and shared. A resident dictionary does not need
it.
+ void enable_term_lookup_cache();
+
private:
struct NormsCacheState;
+ struct TermLookupCache;
io::FileReader* reader_ = nullptr;
format::IndexTier tier_ = format::IndexTier::kT1;
bool has_positions_ = false;
@@ -320,18 +327,27 @@ private:
const format::DictBlockReader** out)
const;
Status locate_candidate_dict_block(std::string_view term, bool*
maybe_present,
uint32_t* ordinal) const;
+ // Groups the candidate terms by DICT block. With `kept_results`, a term
whose lookup the term
+ // lookup cache kept is resolved into it and joins no group.
Status collect_batch_lookup_groups(const std::vector<std::string>& terms,
std::vector<BatchLookupCandidate>*
candidates,
- std::vector<BatchLookupGroup>* groups)
const;
+ std::vector<BatchLookupGroup>* groups,
+ std::vector<BatchLookupResult>*
kept_results) const;
static Status resolve_batch_lookup_entries(const std::vector<std::string>&
terms,
const
std::vector<BatchLookupCandidate>& candidates,
const BatchLookupGroup& group,
const format::DictBlockReader&
block_reader,
std::vector<BatchLookupResult>*
results);
+ // The term lookup cache: whether `term`'s lookup is kept (copied into
*out), and keeping one.
+ bool find_kept_lookup(std::string_view term, BatchLookupResult* out) const;
+ void keep_lookup(std::string_view term, bool found, const
format::DictEntry& entry,
+ uint64_t frq_base, uint64_t prx_base) const;
+ void keep_group_lookups(const BatchLookupState& state, const
BatchLookupGroup& group) const;
std::vector<ResidentDictBlock> resident_dict_blocks_;
std::shared_ptr<NormsCacheState> norms_cache_;
size_t norms_reserved_charge_ = 0;
+ std::shared_ptr<TermLookupCache> term_lookups_;
};
} // namespace doris::snii::reader
diff --git a/be/src/storage/index/inverted/snii/snii_index_reader.cpp
b/be/src/storage/index/inverted/snii/snii_index_reader.cpp
index 4aa0dd10936..d26e4fe030c 100644
--- a/be/src/storage/index/inverted/snii/snii_index_reader.cpp
+++ b/be/src/storage/index/inverted/snii/snii_index_reader.cpp
@@ -111,6 +111,9 @@ Status SniiIndexReader::_get_logical_reader(
return Status::OK();
}
+ // A reader the searcher cache keeps serves later queries, so it keeps
their hot terms' lookups
+ // too; memory_usage() then covers that cache at its fullest.
+ opened_reader->enable_term_lookup_cache();
const size_t reader_size = std::max<size_t>(opened_reader->memory_usage(),
1);
auto* cache_value = new
SniiLogicalReaderCacheValue(std::move(opened_reader), reader_size,
UnixMillis(),
_index_file_reader);
diff --git a/be/test/storage/index/inverted/snii/snii_query_test.cpp
b/be/test/storage/index/inverted/snii/snii_query_test.cpp
index a69ca4457a6..5510699b79f 100644
--- a/be/test/storage/index/inverted/snii/snii_query_test.cpp
+++ b/be/test/storage/index/inverted/snii/snii_query_test.cpp
@@ -418,6 +418,244 @@ TEST(SniiSegmentReaderTest,
LogicalIndexOpenCachesResidentMetadataAndSmallHeader
EXPECT_EQ(file.read_bytes(), 0);
}
+// Writes `input` and opens it; the caller sets SNII_DICT_RESIDENT_MAX=0 so
its DICT blocks are
+// read on demand.
+void open_on_demand_index(const writer::SniiIndexInput& input, MemoryFile*
file,
+ reader::SniiSegmentReader* segment_reader,
+ reader::LogicalIndexReader* index_reader) {
+ writer::SniiCompoundWriter writer(file);
+ assert_ok(writer.add_logical_index(input));
+ assert_ok(writer.finish());
+ assert_ok(reader::SniiSegmentReader::open(file, segment_reader));
+ assert_ok(segment_reader->open_index(input.index_id, input.index_suffix,
index_reader));
+}
+
+// Looks `term` up and returns how many DICT blocks the lookup decoded.
+uint64_t lookup_decodes(const reader::LogicalIndexReader& index_reader,
std::string_view term,
+ bool* found, format::DictEntry* entry = nullptr) {
+ format::DictEntry scratch;
+ uint64_t frq_base = 0;
+ uint64_t prx_base = 0;
+ ::doris::snii::testing::reset_dict_decode_counter();
+ assert_ok(index_reader.lookup(term, found, entry != nullptr ? entry :
&scratch, &frq_base,
+ &prx_base));
+ return ::doris::snii::testing::dict_decode_counter();
+}
+
+// Serves finished index bytes to concurrent readers; MemoryFile records its
reads unguarded.
+class SharedBytesReader final : public doris::snii::io::FileReader {
+public:
+ explicit SharedBytesReader(const std::vector<uint8_t>& bytes) :
_bytes(bytes) {}
+
+ Status read_at(uint64_t offset, size_t len, std::vector<uint8_t>* out)
override {
+ if (offset > _bytes.size() || len > _bytes.size() - offset) {
+ return Status::Corruption("shared bytes read past eof");
+ }
+ out->assign(_bytes.begin() + static_cast<ptrdiff_t>(offset),
+ _bytes.begin() + static_cast<ptrdiff_t>(offset + len));
+ return Status::OK();
+ }
+
+ uint64_t size() const override { return _bytes.size(); }
+
+private:
+ const std::vector<uint8_t>& _bytes;
+};
+
+// A reader that keeps term lookups answers a repeated lookup, whether it
found the term or not,
+// without reading or decoding the DICT block again, and returns the same
entry.
+TEST(SniiTermLookupCacheTest, RepeatedLookupSkipsTheDictBlock) {
+ ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0");
+ // Without a resident filter an absent term reaches its candidate block.
+ ScopedEnv on_demand_bsbf("SNII_BSBF_RESIDENT_MAX", "0");
+ MemoryFile file;
+ reader::SniiSegmentReader segment_reader;
+ reader::LogicalIndexReader index_reader;
+ open_on_demand_index(make_many_term_input(7, "Body", 4096), &file,
&segment_reader,
+ &index_reader);
+ index_reader.enable_term_lookup_cache();
+
+ for (const std::string term : {"term_1002048", "term_1002048x"}) {
+ const bool present = term == "term_1002048";
+ bool found = !present;
+ format::DictEntry first;
+ file.clear_reads();
+ EXPECT_EQ(lookup_decodes(index_reader, term, &found, &first), 1) <<
term;
+ EXPECT_EQ(found, present) << term;
+ EXPECT_GT(file.read_bytes(), 0) << term;
+
+ bool found_again = !present;
+ format::DictEntry again;
+ file.clear_reads();
+ EXPECT_EQ(lookup_decodes(index_reader, term, &found_again, &again), 0)
<< term;
+ EXPECT_EQ(found_again, present) << term;
+ EXPECT_EQ(file.read_bytes(), 0) << term;
+ if (present) {
+ EXPECT_EQ(again.term, first.term);
+ EXPECT_EQ(again.df, first.df);
+ EXPECT_EQ(again.kind, first.kind);
+ EXPECT_EQ(again.frq_off_delta, first.frq_off_delta);
+ EXPECT_EQ(again.frq_len, first.frq_len);
+ EXPECT_EQ(again.frq_bytes, first.frq_bytes);
+ }
+ }
+}
+
+// A batch resolves its kept terms without reads and reads only the blocks of
the others.
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+TEST(SniiTermLookupCacheTest, BatchLookupReadsOnlyTheBlocksOfTermsNotKept) {
+ ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0");
+ MemoryFile file;
+ reader::SniiSegmentReader segment_reader;
+ reader::LogicalIndexReader index_reader;
+ open_on_demand_index(make_many_term_input(7, "Body", 4096), &file,
&segment_reader,
+ &index_reader);
+ index_reader.enable_term_lookup_cache();
+
+ const std::vector<std::string> terms = {"term_1000100", "term_1001200",
"term_1003300"};
+ std::vector<reader::LogicalIndexReader::BatchLookupResult> first;
+ ::doris::snii::testing::reset_dict_decode_counter();
+ assert_ok(index_reader.lookup_batch(terms, &first));
+ EXPECT_EQ(::doris::snii::testing::dict_decode_counter(), terms.size());
+
+ std::vector<reader::LogicalIndexReader::BatchLookupResult> again;
+ ::doris::snii::testing::reset_dict_decode_counter();
+ file.clear_reads();
+ assert_ok(index_reader.lookup_batch(terms, &again));
+ EXPECT_EQ(::doris::snii::testing::dict_decode_counter(), 0);
+ EXPECT_EQ(file.read_bytes(), 0);
+ ASSERT_EQ(again.size(), first.size());
+ for (size_t i = 0; i < terms.size(); ++i) {
+ EXPECT_TRUE(again[i].found) << terms[i];
+ EXPECT_EQ(again[i].entry.term, terms[i]);
+ EXPECT_EQ(again[i].entry.df, first[i].entry.df);
+ EXPECT_EQ(again[i].entry.frq_off_delta, first[i].entry.frq_off_delta);
+ EXPECT_EQ(again[i].frq_base, first[i].frq_base);
+ }
+
+ const std::vector<std::string> mixed = {"term_1000100", "term_1002200",
"term_1003300"};
+ std::vector<reader::LogicalIndexReader::BatchLookupResult> mixed_results;
+ ::doris::snii::testing::reset_dict_decode_counter();
+ assert_ok(index_reader.lookup_batch(mixed, &mixed_results));
+ EXPECT_EQ(::doris::snii::testing::dict_decode_counter(), 1);
+ for (size_t i = 0; i < mixed.size(); ++i) {
+ EXPECT_TRUE(mixed_results[i].found) << mixed[i];
+ EXPECT_EQ(mixed_results[i].entry.term, mixed[i]);
+ }
+}
+
+// The cache keeps at most 32 terms, least recently used out first, keeps no
term longer than
+// 64 bytes, and memory_usage() reserves its whole footprint when it is
enabled.
+TEST(SniiTermLookupCacheTest, KeepsABoundedSetReservedUpFront) {
+ ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0");
+ MemoryFile file;
+ reader::SniiSegmentReader segment_reader;
+ reader::LogicalIndexReader index_reader;
+ writer::SniiIndexInput input = make_many_term_input(7, "Body", 4096);
+ const std::string long_term(65, 'z');
+ input.terms.push_back(make_term(long_term, {{.docid = 4096, .positions =
{0}}}));
+ input.doc_count = 4097;
+ open_on_demand_index(input, &file, &segment_reader, &index_reader);
+
+ const size_t without_cache = index_reader.memory_usage();
+ index_reader.enable_term_lookup_cache();
+ const size_t reserved = index_reader.memory_usage();
+ EXPECT_GT(reserved, without_cache);
+
+ bool found = false;
+ for (uint32_t i = 0; i <= 32; ++i) {
+ EXPECT_EQ(lookup_decodes(index_reader, "term_" +
std::to_string(1000000 + i * 100), &found),
+ 1);
+ EXPECT_TRUE(found);
+ }
+ // The 33rd term pushed out the first; the latest is still kept.
+ EXPECT_EQ(lookup_decodes(index_reader, "term_1003200", &found), 0);
+ EXPECT_EQ(lookup_decodes(index_reader, "term_1000000", &found), 1);
+
+ EXPECT_EQ(lookup_decodes(index_reader, long_term, &found), 1);
+ EXPECT_TRUE(found);
+ EXPECT_EQ(lookup_decodes(index_reader, long_term, &found), 1);
+ EXPECT_EQ(index_reader.memory_usage(), reserved);
+}
+
+// Without the cache every lookup still reads its block; a resident dictionary
never gets one.
+TEST(SniiTermLookupCacheTest, OffByDefaultAndForResidentDictionaries) {
+ bool found = false;
+ {
+ ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0");
+ MemoryFile file;
+ reader::SniiSegmentReader segment_reader;
+ reader::LogicalIndexReader index_reader;
+ open_on_demand_index(make_many_term_input(7, "Body", 4096), &file,
&segment_reader,
+ &index_reader);
+ EXPECT_EQ(lookup_decodes(index_reader, "term_1002048", &found), 1);
+ EXPECT_EQ(lookup_decodes(index_reader, "term_1002048", &found), 1);
+ }
+ ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576");
+ MemoryFile file;
+ reader::SniiSegmentReader segment_reader;
+ reader::LogicalIndexReader index_reader;
+ open_on_demand_index(make_many_term_input(7, "Body", 4096), &file,
&segment_reader,
+ &index_reader);
+ const size_t before = index_reader.memory_usage();
+ index_reader.enable_term_lookup_cache();
+ EXPECT_EQ(index_reader.memory_usage(), before);
+ EXPECT_EQ(lookup_decodes(index_reader, "term_1002048", &found), 0);
+ EXPECT_TRUE(found);
+}
+
+// Queries share one reader: concurrent lookups that keep, find and push out
terms all return the
+// entries a reader without the cache returns.
+TEST(SniiTermLookupCacheTest, ConcurrentLookupsReturnTheUncachedEntries) {
+ ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0");
+ MemoryFile file;
+ reader::SniiSegmentReader segment_reader;
+ reader::LogicalIndexReader plain_reader;
+ open_on_demand_index(make_many_term_input(7, "Body", 4096), &file,
&segment_reader,
+ &plain_reader);
+
+ std::vector<std::string> terms;
+ std::vector<format::DictEntry> expected;
+ for (uint32_t i = 0; i < 64; ++i) {
+ terms.push_back("term_" + std::to_string(1000000 + i * 61));
+ bool found = false;
+ expected.emplace_back();
+ lookup_decodes(plain_reader, terms.back(), &found, &expected.back());
+ ASSERT_TRUE(found);
+ }
+
+ SharedBytesReader shared(file.data());
+ reader::SniiSegmentReader shared_segment;
+ reader::LogicalIndexReader cached_reader;
+ assert_ok(reader::SniiSegmentReader::open(&shared, &shared_segment));
+ assert_ok(shared_segment.open_index(7, "Body", &cached_reader));
+ cached_reader.enable_term_lookup_cache();
+
+ std::atomic<int> mismatches {0};
+ std::vector<std::thread> threads;
+ for (int t = 0; t < 8; ++t) {
+ threads.emplace_back([&, t] {
+ for (int i = 0; i < 2000; ++i) {
+ const size_t k = static_cast<size_t>(t * 7 + i) % terms.size();
+ bool found = false;
+ format::DictEntry entry;
+ uint64_t frq_base = 0;
+ uint64_t prx_base = 0;
+ if (!cached_reader.lookup(terms[k], &found, &entry, &frq_base,
&prx_base).ok() ||
+ !found || entry.term != expected[k].term || entry.df !=
expected[k].df ||
+ entry.frq_off_delta != expected[k].frq_off_delta ||
+ entry.frq_bytes != expected[k].frq_bytes) {
+ ++mismatches;
+ }
+ }
+ });
+ }
+ for (auto& thread : threads) {
+ thread.join();
+ }
+ EXPECT_EQ(mismatches.load(), 0);
+}
+
TEST(SniiPhraseQueryTest, WindowedPhraseQueryKeepsCorrectCandidateOrdinals) {
MemoryFile file;
reader::SniiSegmentReader segment_reader;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
