This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c31cd7f1cb8 [improvement](be) Read SNII dictionary keys in place in
walks and block opens (#69887)
c31cd7f1cb8 is described below
commit c31cd7f1cb8f3bf4847485baa762ac218a79a9e6
Author: Jack <[email protected]>
AuthorDate: Mon Nov 2 14:15:22 2026 +0800
[improvement](be) Read SNII dictionary keys in place in walks and block
opens (#69887)
### What problem does this PR solve?
Issue Number: None
Related PR: #69600
Problem Summary: A regexp without a literal prefix, such as `output
MATCH_REGEXP 'transi.*'`,
expands by walking every term of the SNII dictionary and testing each
key against the pattern. On
a 99M-row table, about 40% of the SNII stack samples of that query were
spent decoding dictionary
entries, twice as much as the pattern match:
- `visit_prefix_range` and `visit_term_range` decoded every key into a
fresh `DictEntry`: the whole
struct reset, a new `std::string`, and a `Status` for every varint.
- `DictBlockReader::open` fully decoded every anchor of the block, about
1 entry in 16, only to get
its key. Every block open pays for this, exact term lookups included.
Both now read an entry's length and front-coded key on the raw cursor. A
walk rebuilds the key in
one buffer it keeps across entries and a rejected key skips its body by
the entry length; `open()`
reads only the keys of the anchors. An entry that does not read cleanly
goes to the checked
decoders, which report what is wrong with it as before, and a walk still
decodes the body of every
entry it accepts with the checked decoder. The on-disk format and the
write path do not change.
`open()` no longer decodes the bodies of the anchors. The block's
crc32c, verified just before,
covers them, and they are checked when they are decoded, like the body
of every other entry.
| Query (99M rows) | Hot, new/previous | Cold, new/previous |
|---|---|---|
| regexp `'transi.*'` | 0.69 | 0.77 |
| regexp that matches no term | 0.67 | |
| regexp `'.*ing'` | 0.72 | |
| regexp `'e.*'` (stops at the expansion limit) | 0.90 | |
| phrase query, term lookup cache off | 0.78 | |
| phrase query, term lookup cache on | 0.99 | 1.01 |
| AND of two terms | 0.99 | 1.01 |
| the other 22 queries | around 1.0 (reruns of those above 1.05:
0.94-1.03) | |
Hot: both versions interleaved in one BE process. Cold: BE restarted and
the index files evicted
before every run, configuration order alternated, 5 runs each. The build
also had #69879 (the term
lookup cache) and #69882. With that cache off, which is master today,
queries open dictionary
blocks more often: the phrase query gains as the 0.78 row shows, and the
five other lookup-heavy
queries measured that way stayed at 0.96-1.04.
---
.../index/inverted/snii/format/dict_block.cpp | 212 ++++++++++++------
.../index/inverted/snii/format/dict_block_test.cpp | 242 +++++++++++++++++++++
2 files changed, 383 insertions(+), 71 deletions(-)
diff --git a/be/src/storage/index/inverted/snii/format/dict_block.cpp
b/be/src/storage/index/inverted/snii/format/dict_block.cpp
index 750a9f06014..96d76e9cfb7 100644
--- a/be/src/storage/index/inverted/snii/format/dict_block.cpp
+++ b/be/src/storage/index/inverted/snii/format/dict_block.cpp
@@ -19,6 +19,10 @@
#include <algorithm>
#include <atomic>
+#include <limits>
+#include <string>
+#include <string_view>
+#include <utility>
#include "storage/index/inverted/snii/encoding/byte_source.h"
#include "storage/index/inverted/snii/encoding/crc32c.h"
@@ -308,6 +312,107 @@ Status check_flags(uint8_t flags, bool has_positions) {
return Status::OK();
}
+// A LEB128 varint of at most five bytes and at most `max`, on the raw cursor.
Within five bytes
+// nothing is shifted out, so whatever this accepts the checked decoders
accept with the same value;
+// anything else, longer encodings included, returns false and is left to them.
+bool read_short_varint(const uint8_t** p, const uint8_t* end, uint64_t max,
uint64_t* v) {
+ uint64_t value = 0;
+ for (uint32_t shift = 0; shift < 35; shift += 7) {
+ if (*p == end) {
+ return false;
+ }
+ const uint8_t byte = *(*p)++;
+ value |= static_cast<uint64_t>(byte & 0x7F) << shift;
+ if (byte < 0x80) {
+ *v = value;
+ return value <= max;
+ }
+ }
+ return false;
+}
+
+// Where an entry lies in its ByteSource.
+struct EntryExtent {
+ size_t body_start = 0;
+ uint64_t total = 0; // entry_len
+ const uint8_t* end = nullptr;
+};
+
+// Reads one entry's length and front-coded key on the raw cursor, rebuilding
`key` from its
+// previous value in place: no DictEntry, no allocation once `key` has grown,
no Status per field. A
+// walk over a whole dictionary does this for every entry, and opening a block
for every anchor. On
+// anything it does not read cleanly it returns false and changes nothing, and
the caller decodes
+// that entry with the checked decoders, which report what is wrong with it.
+bool read_entry_key_fast(ByteSource* src, std::string* key, EntryExtent*
extent) {
+ const uint8_t* p = src->cursor();
+ const uint8_t* const end = src->end();
+ uint64_t total = 0;
+ if (!read_short_varint(&p, end, std::numeric_limits<uint64_t>::max(),
&total) ||
+ std::cmp_greater(total, end - p)) {
+ return false;
+ }
+ const uint8_t* const body = p;
+ const uint8_t* const entry_end = p + total;
+ uint64_t prefix = 0;
+ uint64_t suffix_len = 0;
+ if (!read_short_varint(&p, entry_end,
std::numeric_limits<uint32_t>::max(), &prefix) ||
+ !read_short_varint(&p, entry_end,
std::numeric_limits<uint32_t>::max(), &suffix_len) ||
+ prefix > key->size() || std::cmp_greater(suffix_len, entry_end - p)) {
+ return false;
+ }
+ key->resize(prefix);
+ key->append(reinterpret_cast<const char*>(p), suffix_len);
+ *extent = {.body_start = src->position() + static_cast<size_t>(body -
src->cursor()),
+ .total = total,
+ .end = entry_end};
+ src->set_cursor(p + suffix_len);
+ return true;
+}
+
+enum class WalkStep { kSkip, kStop, kVisit };
+
+// Walks the entries of `src` in key order. `classify(key)` skips an entry,
stops before it (the
+// range is exhausted: *exhausted = true) or hands it to `on_hit`; only handed
entries have their
+// bodies decoded. The key is rebuilt in one reused buffer, which also carries
the front-coding base
+// from entry to entry (an anchor segment starts from an empty one).
+template <typename Classify>
+Status walk_entries(ByteSource* src, IndexTier tier, const Classify& classify,
+ const std::function<Status(DictEntry&&, bool*)>& on_hit,
bool* exhausted) {
+ std::string key;
+ while (!src->eof()) {
+ EntryExtent extent;
+ const bool fast = read_entry_key_fast(src, &key, &extent);
+ if (!fast) {
+ DictEntry checked;
+ RETURN_IF_ERROR(
+ decode_dict_entry_key(src, key, &checked,
&extent.body_start, &extent.total));
+ key = std::move(checked.term);
+ }
+ const WalkStep step = classify(std::string_view(key));
+ if (step == WalkStep::kStop) {
+ *exhausted = true;
+ return Status::OK();
+ }
+ if (step == WalkStep::kSkip) {
+ if (fast) {
+ src->set_cursor(extent.end);
+ } else {
+ RETURN_IF_ERROR(skip_dict_entry_body(src, extent.body_start,
extent.total));
+ }
+ continue;
+ }
+ DictEntry entry;
+ RETURN_IF_ERROR(decode_dict_entry_rest(src, tier, extent.body_start,
extent.total, &entry));
+ entry.term = key;
+ bool stop = false;
+ RETURN_IF_ERROR(on_hit(std::move(entry), &stop));
+ if (stop) {
+ return Status::OK();
+ }
+ }
+ return Status::OK();
+}
+
} // namespace
//
NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
-- Keep invariant-sensitive state transitions together.
@@ -391,10 +496,15 @@ Status DictBlockReader::open(Slice block, IndexTier tier,
bool has_positions,
}
out->anchor_offsets_[i] = off;
// Anchor entries are encoded with prev_term="" and can be decoded
independently to retrieve their term.
+ // Only the key is needed here: the crc above covers the body, which
is checked when it is
+ // decoded, as for any other entry.
ByteSource e_src(covered.subslice(off, anchor_table_begin - off));
- DictEntry probe;
- RETURN_IF_ERROR(decode_dict_entry(&e_src, std::string_view {}, tier,
&probe));
- out->anchor_terms_[i] = std::move(probe.term);
+ EntryExtent extent;
+ if (!read_entry_key_fast(&e_src, &out->anchor_terms_[i], &extent)) {
+ DictEntry probe;
+ RETURN_IF_ERROR(decode_dict_entry(&e_src, std::string_view {},
tier, &probe));
+ out->anchor_terms_[i] = std::move(probe.term);
+ }
}
return Status::OK();
}
@@ -542,7 +652,7 @@ Status DictBlockReader::visit_prefix_range(std::string_view
prefix,
// Scan from the chosen anchor to the end of the entries region: the prefix
// range may span several anchor segments. Anchor entries are encoded with
- // prefix_len=0, so a single running `prev` reconstructs every term
correctly
+ // prefix_len=0, so a single running key reconstructs every term correctly
// even as the scan crosses segment boundaries.
const size_t seg_begin = anchor_offsets_[anchor_idx];
const size_t entries_end =
@@ -553,42 +663,21 @@ Status
DictBlockReader::visit_prefix_range(std::string_view prefix,
}
ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin));
- std::string prev; // the chosen anchor segment starts at an anchor
(prev="")
- while (!src.eof()) {
- DictEntry e;
- size_t body_start = 0;
- uint64_t entry_total = 0;
- RETURN_IF_ERROR(
- decode_dict_entry_key(&src, std::string_view(prev), &e,
&body_start, &entry_total));
- const std::string_view t(e.term);
- if (t < prefix) {
- // Still before the range (only reachable inside the anchor segment
- // that straddles prefix): skip the body, keep the front-coding
base.
- RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start,
entry_total));
- prev = std::move(e.term);
- continue;
- }
- const bool has_prefix = t.size() >= prefix.size() &&
t.starts_with(prefix);
- if (!has_prefix) {
- *prefix_exhausted = true; // sorted: no further matches here or
later
- return Status::OK();
- }
- if (accept_key && !accept_key(t)) {
- // Key-only rejection: never pay for this entry's body.
- RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start,
entry_total));
- prev = std::move(e.term);
- continue;
- }
- // Accepted: materialize the body and hand the entry to the visitor.
- RETURN_IF_ERROR(decode_dict_entry_rest(&src, tier_, body_start,
entry_total, &e));
- prev = e.term; // copy the key before the entry is moved into on_hit
- bool stop = false;
- RETURN_IF_ERROR(on_hit(std::move(e), &stop));
- if (stop) {
- return Status::OK();
- }
- }
- return Status::OK();
+ return walk_entries(
+ &src, tier_,
+ [&](std::string_view term) {
+ // Still before the range: only inside the anchor segment that
straddles prefix.
+ if (term < prefix) {
+ return WalkStep::kSkip;
+ }
+ // Sorted: no further matches here or later.
+ if (!term.starts_with(prefix)) {
+ return WalkStep::kStop;
+ }
+ // Key-only rejection: never pay for this entry's body.
+ return accept_key && !accept_key(term) ? WalkStep::kSkip :
WalkStep::kVisit;
+ },
+ on_hit, prefix_exhausted);
}
Status DictBlockReader::visit_term_range(std::string_view lower_inclusive,
@@ -622,37 +711,18 @@ Status DictBlockReader::visit_term_range(std::string_view
lower_inclusive,
}
ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin));
- std::string prev;
- while (!src.eof()) {
- DictEntry entry;
- size_t body_start = 0;
- uint64_t entry_total = 0;
- RETURN_IF_ERROR(decode_dict_entry_key(&src, std::string_view(prev),
&entry, &body_start,
- &entry_total));
- const std::string_view term(entry.term);
- if (term < lower_inclusive) {
- RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start,
entry_total));
- prev = std::move(entry.term);
- continue;
- }
- if (upper_exclusive.has_value() && term >= *upper_exclusive) {
- *range_exhausted = true;
- return Status::OK();
- }
- if (accept_key && !accept_key(term)) {
- RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start,
entry_total));
- prev = std::move(entry.term);
- continue;
- }
- RETURN_IF_ERROR(decode_dict_entry_rest(&src, tier_, body_start,
entry_total, &entry));
- prev = entry.term;
- bool stop = false;
- RETURN_IF_ERROR(on_hit(std::move(entry), &stop));
- if (stop) {
- return Status::OK();
- }
- }
- return Status::OK();
+ return walk_entries(
+ &src, tier_,
+ [&](std::string_view term) {
+ if (term < lower_inclusive) {
+ return WalkStep::kSkip;
+ }
+ if (upper_exclusive.has_value() && term >= *upper_exclusive) {
+ return WalkStep::kStop;
+ }
+ return accept_key && !accept_key(term) ? WalkStep::kSkip :
WalkStep::kVisit;
+ },
+ on_hit, range_exhausted);
}
} // namespace doris::snii::format
diff --git a/be/test/storage/index/inverted/snii/format/dict_block_test.cpp
b/be/test/storage/index/inverted/snii/format/dict_block_test.cpp
index 1effa3621ee..c4a2ab4f8f8 100644
--- a/be/test/storage/index/inverted/snii/format/dict_block_test.cpp
+++ b/be/test/storage/index/inverted/snii/format/dict_block_test.cpp
@@ -19,7 +19,10 @@
#include <gtest/gtest.h>
+#include <algorithm>
+#include <functional>
#include <limits>
+#include <optional>
#include <string>
#include <string_view>
#include <vector>
@@ -27,6 +30,7 @@
#include "common/status.h"
#include "storage/index/inverted/snii/common/slice.h"
#include "storage/index/inverted/snii/encoding/byte_sink.h"
+#include "storage/index/inverted/snii/encoding/byte_source.h"
#include "storage/index/inverted/snii/encoding/crc32c.h"
#include "storage/index/inverted/snii/format/dict_entry.h"
#include "storage/index/inverted/snii/format/format_constants.h"
@@ -605,3 +609,241 @@ TEST(SniiDictBlockTest,
FinishUnaffectedByDeadPrevTermRemoval) {
EXPECT_EQ(all[i].term, terms[i]) << i; // front-coding base rebuilt
without prev_term_
}
}
+
+namespace {
+
+// Pod-ref and inline entries whose keys run from a few bytes to tens, sharing
long prefixes.
+std::vector<DictEntry> MakeWalkEntries(int count) {
+ std::vector<DictEntry> entries;
+ for (int i = 0; i < count; ++i) {
+ std::string term =
+ "k" + std::string(static_cast<size_t>(i % 7) * 9, 'p') +
std::to_string(100000 + i);
+ entries.push_back(i % 3 == 0 ? MakeInline(std::move(term),
static_cast<uint32_t>(i + 1))
+ : MakePodRef(std::move(term),
static_cast<uint32_t>(i + 1),
+ static_cast<uint64_t>(i) *
10));
+ }
+ std::ranges::sort(entries, {}, &DictEntry::term);
+ return entries;
+}
+
+uint32_t LoadLe32(const std::vector<uint8_t>& bytes, size_t at) {
+ uint32_t v = 0;
+ for (size_t k = 0; k < 4; ++k) {
+ v |= static_cast<uint32_t>(bytes[at + k]) << (8 * k);
+ }
+ return v;
+}
+
+// Re-stamps the crc32c footer after a test edits a block, so that the edit
gets past the crc.
+void RestampCrc(std::vector<uint8_t>* bytes) {
+ const size_t n = bytes->size();
+ const uint32_t crc = doris::snii::crc32c(Slice(bytes->data(), n - 4));
+ for (size_t k = 0; k < 4; ++k) {
+ (*bytes)[n - 4 + k] = static_cast<uint8_t>(crc >> (8 * k));
+ }
+}
+
+// A block of three pod-ref entries that are all anchors, and the offset of
the second one:
+// [entry_len][prefix_len][suffix_len]["bbb"][flags][df][locator].
+std::vector<uint8_t> BuildAnchorsBlock(size_t* second) {
+ const std::vector<DictEntry> entries = {MakePodRef("aaa", 1, 0),
MakePodRef("bbb", 2, 10),
+ MakePodRef("ccc", 3, 20)};
+ std::vector<uint8_t> bytes = BuildBlock(entries, IndexTier::kT1, false,
100, 0, 1);
+ *second = LoadLe32(bytes, bytes.size() - 8 - 2 * 4); // [offsets u32 x
3][n_anchors][crc32c]
+ return bytes;
+}
+
+// The walk as it was before keys were read on a raw cursor: every entry
through the checked
+// decoders. The walks must report exactly what this reports, on valid and
corrupt blocks alike.
+Status CheckedWalk(Slice entries, std::optional<std::string_view> upper,
+ const std::function<bool(std::string_view)>& accept,
+ std::vector<std::string>* visited, bool* exhausted) {
+ ByteSource src(entries);
+ std::string prev;
+ while (!src.eof()) {
+ DictEntry entry;
+ size_t body_start = 0;
+ uint64_t total = 0;
+ RETURN_IF_ERROR(decode_dict_entry_key(&src, prev, &entry, &body_start,
&total));
+ if (upper.has_value() && entry.term >= *upper) {
+ *exhausted = true;
+ return Status::OK();
+ }
+ if (accept(entry.term)) {
+ RETURN_IF_ERROR(
+ decode_dict_entry_rest(&src, IndexTier::kT2, body_start,
total, &entry));
+ visited->push_back(entry.term);
+ } else {
+ RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, total));
+ }
+ prev = std::move(entry.term);
+ }
+ return Status::OK();
+}
+
+// A walk's on_hit that keeps every entry it is handed.
+std::function<Status(DictEntry&&, bool*)> Collect(std::vector<DictEntry>* got)
{
+ return [got](DictEntry&& entry, bool*) {
+ got->push_back(std::move(entry));
+ return Status::OK();
+ };
+}
+
+// `got` must be the entries of `all` that `wanted` picks, in order and with
their bodies.
+void ExpectVisited(const std::vector<DictEntry>& all, const
std::vector<DictEntry>& got,
+ const std::function<bool(const DictEntry&)>& wanted) {
+ size_t next = 0;
+ for (const DictEntry& want : all) {
+ if (!wanted(want)) {
+ continue;
+ }
+ ASSERT_LT(next, got.size());
+ ExpectCommon(got[next], want);
+ EXPECT_EQ(got[next].frq_off_delta, want.frq_off_delta);
+ EXPECT_EQ(got[next].frq_bytes, want.frq_bytes);
+ ++next;
+ }
+ EXPECT_EQ(next, got.size());
+}
+
+// Walks the block both ways and compares what they report; counts the walks
that found corruption.
+void ExpectWalkAgrees(const DictBlockReader& reader, Slice entries,
+ std::optional<std::string_view> upper,
+ const std::function<bool(std::string_view)>& accept,
size_t* corrupt) {
+ std::vector<std::string> want;
+ bool want_exhausted = false;
+ const Status want_st = CheckedWalk(entries, upper, accept, &want,
&want_exhausted);
+ std::vector<DictEntry> got;
+ bool got_exhausted = false;
+ const Status got_st = reader.visit_term_range("", upper, accept,
Collect(&got), &got_exhausted);
+ EXPECT_EQ(got_st.to_string(), want_st.to_string());
+ ASSERT_EQ(got.size(), want.size());
+ for (size_t i = 0; i < got.size(); ++i) {
+ EXPECT_EQ(got[i].term, want[i]);
+ }
+ EXPECT_EQ(got_exhausted, want_exhausted);
+ *corrupt += want_st.ok() ? 0 : 1;
+}
+
+} // namespace
+
+// The walks rebuild each key in one reused buffer. Short and long keys, long
prefixes shared across
+// anchors, and inline and pod-ref bodies all come out as decode_all() reads
them, for any prefix,
+// range and key filter.
+TEST(SniiDictBlock, WalksMatchDecodeAll) {
+ const std::vector<uint8_t> bytes =
+ BuildBlock(MakeWalkEntries(300), IndexTier::kT2, true, 8192,
16384, 8);
+ DictBlockReader reader;
+ ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT2, true,
&reader).ok());
+ std::vector<DictEntry> all;
+ ASSERT_TRUE(reader.decode_all(&all).ok());
+ ASSERT_EQ(all.size(), 300U);
+
+ const std::vector<std::function<bool(std::string_view)>> filters = {
+ {},
+ [](std::string_view term) { return !term.empty() && term.back() %
2 == 0; },
+ [](std::string_view term) { return term.size() > 20; },
+ [](std::string_view) { return false; }};
+ for (const std::string_view prefix :
+ {"", "k", "kppppppppp", "kpppppppppppppppppp1002", "k1001", "zzz"}) {
+ for (size_t f = 0; f < filters.size(); ++f) {
+ SCOPED_TRACE(std::string(prefix) + " filter " + std::to_string(f));
+ const auto& filter = filters[f];
+ std::vector<DictEntry> got;
+ bool exhausted = false;
+ ASSERT_TRUE(reader.visit_prefix_range(prefix, filter,
Collect(&got), &exhausted).ok());
+ ExpectVisited(all, got, [&](const DictEntry& entry) {
+ return entry.term.starts_with(prefix) && (!filter ||
filter(entry.term));
+ });
+ }
+ }
+
+ const std::string& lower = all[37].term;
+ const std::string& upper = all[201].term;
+ std::vector<DictEntry> got;
+ bool exhausted = false;
+ ASSERT_TRUE(reader.visit_term_range(lower, std::string_view(upper),
filters[1], Collect(&got),
+ &exhausted)
+ .ok());
+ ExpectVisited(all, got, [&](const DictEntry& entry) {
+ return entry.term >= lower && entry.term < upper &&
filters[1](entry.term);
+ });
+ EXPECT_TRUE(exhausted);
+}
+
+// Every one-byte change to the entries of a block, with the crc re-stamped so
that open() accepts
+// it, leaves a walk reporting exactly what a walk over the checked decoders
reports: the same
+// status, keys and end of range. An entry whose length or key the raw-cursor
reader cannot read
+// cleanly goes through the checked decoders, so its corruption is still
reported.
+TEST(SniiDictBlock, WalksAgreeWithCheckedDecodersOnCorruptBlocks) {
+ const std::vector<DictEntry> entries = MakeWalkEntries(24);
+ const std::vector<uint8_t> good = BuildBlock(entries, IndexTier::kT2,
true, 8192, 16384, 16);
+ // Tail: [anchor offsets u32 x n][n_anchors u32][crc32c u32]; the first
offset starts the entries.
+ const size_t n = good.size();
+ const size_t entries_end = n - 8 - 4 * static_cast<size_t>(LoadLe32(good,
n - 8));
+ const size_t entries_begin = LoadLe32(good, entries_end);
+ const std::optional<std::string_view> upper =
std::string_view(entries[19].term);
+ const std::vector<std::function<bool(std::string_view)>> filters = {
+ [](std::string_view) { return true; }, [](std::string_view) {
return false; },
+ [](std::string_view term) { return !term.empty() && term.back() %
2 == 0; }};
+
+ size_t corrupt = 0;
+ for (size_t at = entries_begin; at < entries_end; ++at) {
+ // A changed low bit, a larger value, and a varint that runs on into
the next byte.
+ for (const uint8_t mask : {0x01, 0x40, 0x80}) {
+ std::vector<uint8_t> bytes = good;
+ bytes[at] ^= mask;
+ RestampCrc(&bytes);
+ DictBlockReader reader;
+ if (!DictBlockReader::open(Slice(bytes), IndexTier::kT2, true,
&reader).ok()) {
+ continue; // open() decodes the anchor entries itself
+ }
+ SCOPED_TRACE("byte " + std::to_string(at) + " ^ " +
std::to_string(mask));
+ const Slice region(bytes.data() + entries_begin, entries_end -
entries_begin);
+ for (const auto& filter : filters) {
+ ExpectWalkAgrees(reader, region, std::nullopt, filter,
&corrupt);
+ ExpectWalkAgrees(reader, region, upper, filter, &corrupt);
+ }
+ ASSERT_FALSE(HasFailure());
+ }
+ }
+ EXPECT_GT(corrupt, 0U); // the changes do reach the checked decoders'
errors
+}
+
+// open() reads only the key of each anchor, and an anchor key it cannot read
is still an error.
+TEST(SniiDictBlock, OpenRejectsUnreadableAnchorKeys) {
+ size_t second = 0;
+ const std::vector<uint8_t> good = BuildAnchorsBlock(&second);
+ ASSERT_EQ(good[second + 1], 0); // an anchor has no previous key to share
a prefix with
+
+ auto expect_rejected = [&](size_t at, uint8_t value, const char* what) {
+ std::vector<uint8_t> bytes = good;
+ bytes[at] = value;
+ RestampCrc(&bytes);
+ DictBlockReader reader;
+ const Status st = DictBlockReader::open(Slice(bytes), IndexTier::kT1,
false, &reader);
+ EXPECT_TRUE(st.is<doris::ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>())
+ << what << ": " << st.to_string();
+ };
+ expect_rejected(second + 1, 1, "front-coded anchor");
+ expect_rejected(second, 0x7F, "entry past the entries");
+ expect_rejected(second, 1, "key past its entry");
+ expect_rejected(second + 2, 0x7F, "key past the entries");
+}
+
+// An anchor's body is checked when it is decoded, as any other entry's:
open() does not read it.
+TEST(SniiDictBlock, AnchorBodyIsCheckedWhenDecoded) {
+ size_t second = 0;
+ std::vector<uint8_t> bytes = BuildAnchorsBlock(&second);
+ bytes[second + 7] |= 0x80; // df runs on into the locator
+ RestampCrc(&bytes);
+
+ DictBlockReader reader;
+ ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false,
&reader).ok());
+ bool found = false;
+ DictEntry entry;
+ ASSERT_TRUE(reader.find_term("aaa", &found, &entry).ok());
+ EXPECT_TRUE(found);
+ const Status st = reader.find_term("bbb", &found, &entry);
+ EXPECT_TRUE(st.is<doris::ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>()) <<
st.to_string();
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
