This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8db05407dcf [fix](inverted index) Stream compact docs-only SNII
postings (#69192)
8db05407dcf is described below
commit 8db05407dcf574c90da47579acb0ad7af57e027a
Author: Jack <[email protected]>
AuthorDate: Sun Oct 18 10:27:19 2026 +0800
[fix](inverted index) Stream compact docs-only SNII postings (#69192)
### What problem does this PR solve?
Issue Number: CIR-22263
Problem Summary: SNII streamed compaction rejects compatible docs-only
(T1) postings when the destination does not request phrase positions,
then rebuilds those indexes from raw columns. This PR merges compatible
T1 postings directly and retains raw rebuild for mixed or incompatible
sources. Gram sources also retain raw rebuild because they can omit
common postings and need their gram scheme preserved.
Cloud compaction can assign nonzero physical destination segment IDs.
The streamed SNII path now looks up its destination writer by physical
ID, matching the existing CLucene path; looking it up by ordinal caused
a BE crash in Cloud P0.
This removes one repeated fallback seen in the CIR logs. The main T2
merge hotspot for tokenized indexes remains a separate performance
question. Phrase-position defaults are handled in #69199.
### Release note
Compatible docs-only SNII indexes use streamed compaction.
### Check List (For Author)
- Test
- [x] Unit Test: 17 SNII eligibility tests and the nonzero-segment-ID
compaction test passed on the updated commit; 18 compaction-focused BE
tests passed before the follow-up fix.
- [x] Regression test: `test_storage_format_snii` passed on the combined
source tree before this PR was split.
- [x] Manual test: complete FE and BE builds and local health checks
passed before the split.
- Behavior changed:
- [x] Yes. Compatible T1 postings are streamed; gram sources rebuild
from raw columns.
- Does this need documentation?
- [x] No.
The complete PR diff passes the English description check and
clang-format 16 on the merged target tree. The updated commit is
awaiting remote CI.
---
be/src/storage/compaction/compaction.cpp | 3 +-
.../index/inverted/snii/compaction/eligibility.cpp | 61 ++++++++++++-----
.../index/inverted/snii/compaction/eligibility.h | 16 ++---
.../inverted/snii/compaction/posting_cursor.cpp | 8 +--
.../snii/compaction/snii_index_compaction.cpp | 7 +-
.../snii/compaction/snii_index_compaction.h | 4 +-
.../clucene/compaction/index_compaction_test.cpp | 40 ++++++-----
.../snii_compaction_eligibility_test.cpp | 80 +++++++++++++++++++++-
.../snii/compaction/snii_index_compaction_test.cpp | 52 ++++++++++++++
9 files changed, 214 insertions(+), 57 deletions(-)
diff --git a/be/src/storage/compaction/compaction.cpp
b/be/src/storage/compaction/compaction.cpp
index dc395cbba44..ecd2abf9154 100644
--- a/be/src/storage/compaction/compaction.cpp
+++ b/be/src/storage/compaction/compaction.cpp
@@ -1335,7 +1335,8 @@ Status Compaction::do_inverted_index_compaction() {
break;
}
auto* destination_writer =
-
inverted_index_file_writers[cast_set<int>(destination_ordinal)]
+ inverted_index_file_writers
+
.at(dest_segment_ids[destination_ordinal])
.get();
merge_status =
destination_writer->add_snii_index_streamed(
index_meta,
dest_segment_num_rows[destination_ordinal],
diff --git a/be/src/storage/index/inverted/snii/compaction/eligibility.cpp
b/be/src/storage/index/inverted/snii/compaction/eligibility.cpp
index bff80048d61..928f5ed47cf 100644
--- a/be/src/storage/index/inverted/snii/compaction/eligibility.cpp
+++ b/be/src/storage/index/inverted/snii/compaction/eligibility.cpp
@@ -74,6 +74,26 @@ Status validate_source_shape(const
reader::LogicalIndexReader& source, size_t so
return Status::OK();
}
+Status validate_source_shape(const reader::LogicalIndexReader& source, size_t
source_ordinal,
+ bool has_positions) {
+ if (source.has_positions() != has_positions ||
+ source.tier() != (has_positions ? format::IndexTier::kT2 :
format::IndexTier::kT1)) {
+ return reject(
+ fmt::format("source {} position shape differs from
destination", source_ordinal));
+ }
+ // Gram indexes may omit common postings and need their scheme retained on
output.
+ if (!has_positions && source.gram_scheme().has_value()) {
+ return reject(fmt::format("source {} carries a gram scheme",
source_ordinal));
+ }
+ const auto& stats = source.stats();
+ if (stats.indexed_doc_count > stats.doc_count ||
+ stats.null_count != stats.doc_count - stats.indexed_doc_count) {
+ return reject(fmt::format("source {} document statistics violate
indexed + null = doc",
+ source_ordinal));
+ }
+ return Status::OK();
+}
+
Status reject_legacy_bigram(const reader::LogicalIndexReader& source, size_t
source_ordinal) {
bool found_legacy_bigram = false;
RETURN_IF_ERROR(source.visit_prefix_terms(
@@ -117,8 +137,9 @@ Status resolve_destination_analyzer(const TabletIndex&
destination_index,
Status validate_destination_policy(const TabletIndex& destination_index,
const AnalyzerProviderFactory&
analyzer_provider_factory) {
const auto& properties = destination_index.properties();
- if (get_parser_phrase_support_string_from_properties(properties) !=
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+ if (properties.contains(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY) &&
+ get_parser_phrase_support_string_from_properties(properties) !=
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
return reject("destination does not request phrase positions");
}
if (!inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)) {
@@ -146,8 +167,10 @@ Status validate_plain_t2_source_eligibility(const
reader::LogicalIndexReader& so
Status validate_snii_source_eligibility(const reader::LogicalIndexReader&
source,
size_t source_ordinal,
- const SniiCompactionEligibility&
/*eligibility*/) {
- return validate_plain_t2_source_eligibility(source, source_ordinal);
+ const SniiCompactionEligibility&
eligibility) {
+ RETURN_IF_ERROR(
+ validate_source_shape(source, source_ordinal,
eligibility.destination_has_positions));
+ return reject_legacy_bigram(source, source_ordinal);
}
Status validate_plain_t2_compaction_eligibility(
@@ -206,10 +229,14 @@ Status validate_snii_compaction_eligibility(
if (!destination_index.is_inverted_index()) {
return reject("destination is not an inverted index");
}
- if
(get_parser_phrase_support_string_from_properties(destination_index.properties())
!=
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
- return reject("destination does not request phrase positions");
- }
+ const auto& destination_properties = destination_index.properties();
+ const bool explicit_phrase_policy =
+
destination_properties.contains(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY);
+ out->destination_has_positions =
+ explicit_phrase_policy
+ ?
get_parser_phrase_support_string_from_properties(destination_properties) ==
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES
+ : sources.front().reader.get().has_positions();
const TabletIndex& first_index_meta = sources.front().index_meta.get();
if (!first_index_meta.is_inverted_index()) {
@@ -238,18 +265,18 @@ Status validate_snii_compaction_eligibility(
}
for (size_t source_ordinal = 0; source_ordinal < sources.size();
++source_ordinal) {
- if (sources[source_ordinal].reader.get().tier() !=
format::IndexTier::kT2) {
- return reject(fmt::format("source {} is not a positional (T2)
index", source_ordinal));
- }
-
RETURN_IF_ERROR(validate_plain_t2_source_eligibility(sources[source_ordinal].reader.get(),
- source_ordinal));
+
RETURN_IF_ERROR(validate_snii_source_eligibility(sources[source_ordinal].reader.get(),
+ source_ordinal,
*out));
+ }
+ if (out->destination_has_positions) {
+ RETURN_IF_ERROR(validate_destination_policy(destination_index,
analyzer_provider_factory));
}
- RETURN_IF_ERROR(validate_destination_policy(destination_index,
analyzer_provider_factory));
// Merged norms are rebuilt from the postings, so the destination follows
the same norms
// policy as a fresh write whether or not the sources carry norms.
- out->destination_writes_norms =
inverted_index::InvertedIndexAnalyzer::should_analyzer(
- destination_index.properties()) &&
-
should_write_index_norms(destination_index);
+ out->destination_writes_norms =
+ out->destination_has_positions &&
+
inverted_index::InvertedIndexAnalyzer::should_analyzer(destination_properties)
&&
+ should_write_index_norms(destination_index);
return Status::OK();
}
diff --git a/be/src/storage/index/inverted/snii/compaction/eligibility.h
b/be/src/storage/index/inverted/snii/compaction/eligibility.h
index 8fd5eec28f4..ae6da0090a4 100644
--- a/be/src/storage/index/inverted/snii/compaction/eligibility.h
+++ b/be/src/storage/index/inverted/snii/compaction/eligibility.h
@@ -37,9 +37,7 @@ class LogicalIndexReader;
namespace snii::compaction {
-// One already-opened SNII source logical index and the TabletIndex metadata
-// that produced it. References make null inputs unrepresentable; both objects
-// must outlive validate_plain_t2_compaction_eligibility().
+// One opened SNII source and its index metadata. Both objects must outlive
validation.
struct PlainT2CompactionSource {
std::reference_wrapper<const reader::LogicalIndexReader> reader;
std::reference_wrapper<const TabletIndex> index_meta;
@@ -52,6 +50,8 @@ using AnalyzerProviderFactory =
std::function<segment_v2::inverted_index::Analyz
const InvertedIndexAnalyzerConfig&)>;
struct SniiCompactionEligibility {
+ // When the phrase property is absent, preserve the source posting shape.
+ bool destination_has_positions = true;
// A2: An analyzed destination index with positions must include norms.
The posting merge
// rebuilds them from source term frequencies (sum per document, clamped
to 1..255), so older
// T2 segments without norms can gain norms without reanalysis.
@@ -69,9 +69,7 @@ Status validate_plain_t2_source(const
reader::LogicalIndexReader& source, size_t
Status validate_plain_t2_source_eligibility(const reader::LogicalIndexReader&
source,
size_t source_ordinal);
-// Revalidates one opened source against an already selected streamed shape.
-// CommonGrams checks include exact current versions and static identity
equality
-// with the eligibility seed.
+// Revalidates one source against the selected destination posting shape.
Status validate_snii_source_eligibility(const reader::LogicalIndexReader&
source,
size_t source_ordinal,
const SniiCompactionEligibility&
eligibility);
@@ -91,10 +89,8 @@ Status validate_plain_t2_compaction_eligibility(
std::span<const PlainT2CompactionSource> sources, const TabletIndex&
destination_index,
const AnalyzerProviderFactory& analyzer_provider_factory = {});
-// Accepts either the existing plain positions-only T2 shape or homogeneous,
-// complete CommonGrams T3 sources. Mixed shapes and any CommonGrams identity
or
-// destination-build-policy mismatch return INVERTED_INDEX_NOT_SUPPORTED so the
-// caller can rebuild from raw columns before creating streamed output.
+// Accepts homogeneous T1 or T2 sources when their posting shape matches the
destination.
+// Mixed shapes fall back to rebuilding from raw columns.
Status validate_snii_compaction_eligibility(
std::span<const PlainT2CompactionSource> sources, const TabletIndex&
destination_index,
SniiCompactionEligibility* out,
diff --git a/be/src/storage/index/inverted/snii/compaction/posting_cursor.cpp
b/be/src/storage/index/inverted/snii/compaction/posting_cursor.cpp
index b2999afc036..2ef64724503 100644
--- a/be/src/storage/index/inverted/snii/compaction/posting_cursor.cpp
+++ b/be/src/storage/index/inverted/snii/compaction/posting_cursor.cpp
@@ -579,12 +579,8 @@ Status SniiPostingCursor::init() {
if (!read_context_->failed_status().ok()) {
return read_context_->failed_status();
}
- if (index_->tier() == format::IndexTier::kT1) {
- return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, false>(
- "posting_cursor: positions index is required");
- }
- if (!index_->has_positions()) {
- return posting_corruption("positions tier lacks positions capability",
source_ordinal_);
+ if (index_->has_positions() != term_has_positions_) {
+ return posting_corruption("term position shape differs from source
index", source_ordinal_);
}
if (source_ordinal_ >= rowid_conversion_->source_segment_count()) {
return Status::Error<ErrorCode::INVALID_ARGUMENT, false>(
diff --git
a/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.cpp
b/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.cpp
index a46223caf67..9d74153521f 100644
--- a/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.cpp
+++ b/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.cpp
@@ -315,7 +315,8 @@ writer::TrackedEncodedNorms
SniiPlainT2MergePlan::take_destination_encoded_norms
}
format::IndexConfig SniiPlainT2MergePlan::destination_index_config() const {
- return format::IndexConfig::kDocsPositions;
+ return eligibility_.destination_has_positions ?
format::IndexConfig::kDocsPositions
+ :
format::IndexConfig::kDocsOnly;
}
Status SniiPlainT2MergePlan::poison(Status status) {
@@ -338,8 +339,8 @@ Status
SniiPlainT2MergePlan::take_front_source(TermMergeFrontier* frontier, Curr
}
const bool source_has_positions = posting_entry_has_positions(entry);
- if (!source_has_positions) {
- return index_compaction_merge_corruption("ordinary term has docs-only
posting shape");
+ if (source_has_positions != eligibility_.destination_has_positions) {
+ return index_compaction_merge_corruption("term position shape differs
from destination");
}
if (!current->has_positions.has_value()) {
current->has_positions = source_has_positions;
diff --git
a/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.h
b/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.h
index b0902340ee7..266c239e301 100644
--- a/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.h
+++ b/be/src/storage/index/inverted/snii/compaction/snii_index_compaction.h
@@ -37,8 +37,8 @@ namespace doris::snii::compaction {
class TermMergeFrontier;
-// Prepared, one-shot merge of one plain positions-only logical index across
-// source segments. prepare() performs every O(1)-metadata, row-id and NULL
+// Prepared, one-shot merge of one homogeneous SNII index across source
segments.
+// prepare() performs metadata, row-id and NULL
// preflight before the caller creates destination streamed sessions. execute()
// then performs exactly one dictionary/posting pass and seals every session.
//
diff --git
a/be/test/storage/index/inverted/clucene/compaction/index_compaction_test.cpp
b/be/test/storage/index/inverted/clucene/compaction/index_compaction_test.cpp
index da784588f8f..68b58e71deb 100644
---
a/be/test/storage/index/inverted/clucene/compaction/index_compaction_test.cpp
+++
b/be/test/storage/index/inverted/clucene/compaction/index_compaction_test.cpp
@@ -1813,44 +1813,50 @@ TEST_F(IndexCompactionTest,
snii_native_merge_validates_rowids_once_and_matches_
}
}
-// The design's core split: two logical indexes share one column, the eligible
-// one merges natively (no analyzer), the ineligible one (no phrase positions)
-// raw-builds from the column -- in the SAME compaction pass. The old behavior
-// AND-folded eligibility per column and fell back to raw for both.
-TEST_F(IndexCompactionTest,
snii_native_merge_compacts_eligible_index_and_raw_builds_sibling) {
- DEFER({});
-
+TEST_F(IndexCompactionTest,
snii_native_merge_compacts_phrase_and_docs_only_siblings) {
_build_snii_multi_index_tablet(/*second_supports_phrase=*/false);
const std::vector<RowsetSharedPtr> rowsets = _build_snii_source_rowsets();
- auto check_split = [](const BaseCompaction& compaction, const
RowsetWriterContext& ctx) {
- // Only the phrase-capable index merges; its no-phrase sibling
raw-builds.
+ constexpr int32_t output_segment_start_id = 10;
+ auto check_merge = [](const BaseCompaction& compaction, const
RowsetWriterContext& ctx) {
EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty());
EXPECT_EQ(ctx.snii_indexes_to_do_compaction,
- (std::set<std::pair<int32_t, int64_t>> {{1, 11001}}));
+ (std::set<std::pair<int32_t, int64_t>> {{1, 11001}, {1,
11002}}));
EXPECT_EQ(compaction._output_rowset->num_segments(), 2);
};
RowsetSharedPtr output;
- const Status status = IndexCompactionUtils::do_compaction(rowsets,
_engine_ref, _tablet, true,
- output,
check_split, 1000);
+ const Status status =
+ IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet,
true, output,
+ check_merge, 1000,
output_segment_start_id);
ASSERT_TRUE(status.ok()) << status;
ASSERT_NE(output, nullptr);
std::vector<uint32_t> segment_rows;
output->rowset_meta()->get_num_segment_rows(&segment_rows);
ASSERT_EQ(segment_rows.size(), output->num_segments());
- for (uint32_t segment_id = 0; segment_id < output->num_segments();
++segment_id) {
- const auto segment_path = output->segment_path(segment_id);
+ for (uint32_t segment_pos = 0; segment_pos < output->num_segments();
++segment_pos) {
+ const auto segment = output->segment(segment_pos);
+ EXPECT_EQ(segment.id(), output_segment_start_id + segment_pos);
+ const auto segment_path = segment.path();
ASSERT_TRUE(segment_path.has_value()) << segment_path.error();
auto file_reader = IndexCompactionUtils::init_index_file_reader(
output, segment_path.value(),
InvertedIndexStorageFormatPB::SNII);
for (const TabletIndex* index : _tablet_schema->inverted_indexes()) {
const auto logical_index = file_reader->open_snii_index(index);
EXPECT_TRUE(logical_index.has_value()) << logical_index.error();
- // The merged and the raw-built index both cover every segment row.
EXPECT_EQ(logical_index.value()->stats().doc_count,
- static_cast<uint64_t>(segment_rows[segment_id]))
- << "index " << index->index_id() << " segment " <<
segment_id;
+ static_cast<uint64_t>(segment_rows[segment_pos]))
+ << "index " << index->index_id() << " segment " <<
segment_pos;
}
}
+ RowsetSharedPtr raw_rebuild;
+ const Status rebuild_status = IndexCompactionUtils::do_compaction(
+ rowsets, _engine_ref, _tablet, false, raw_rebuild, nullptr, 1000);
+ ASSERT_TRUE(rebuild_status.ok()) << rebuild_status;
+ ASSERT_NE(raw_rebuild, nullptr);
+ ASSERT_EQ(output->num_segments(), raw_rebuild->num_segments());
+ for (uint32_t segment_pos = 0; segment_pos < output->num_segments();
++segment_pos) {
+ EXPECT_EQ(_read_index_file_bytes(output,
output->segment(segment_pos).id()),
+ _read_index_file_bytes(raw_rebuild,
raw_rebuild->segment(segment_pos).id()));
+ }
}
TEST_F(IndexCompactionTest,
snii_native_merge_aborts_after_partial_destination_creation) {
diff --git
a/be/test/storage/index/inverted/snii/compaction/snii_compaction_eligibility_test.cpp
b/be/test/storage/index/inverted/snii/compaction/snii_compaction_eligibility_test.cpp
index 8b58243c328..ba062c4bfdc 100644
---
a/be/test/storage/index/inverted/snii/compaction/snii_compaction_eligibility_test.cpp
+++
b/be/test/storage/index/inverted/snii/compaction/snii_compaction_eligibility_test.cpp
@@ -32,6 +32,7 @@
#include "common/config.h"
#include "common/status.h"
#include "storage/index/inverted/analysis/analyzer/analyzer_provider.h"
+#include "storage/index/inverted/analysis/gram/gram_scheme.h"
#include "storage/index/inverted/analysis/inverted_index_parser.h"
#include "storage/index/inverted/snii/common/slice.h"
#include "storage/index/inverted/snii/compaction/eligibility.h"
@@ -96,7 +97,9 @@ struct OpenedIndex {
reader::LogicalIndexReader reader;
};
-std::unique_ptr<OpenedIndex> open_index(const IndexShape& shape) {
+std::unique_ptr<OpenedIndex> open_index(
+ const IndexShape& shape,
+ std::optional<doris::segment_v2::gram::GramScheme> gram_scheme =
std::nullopt) {
std::vector<uint8_t> file_bytes;
format::SectionRefs refs;
if (shape.has_norms) {
@@ -122,6 +125,7 @@ std::unique_ptr<OpenedIndex> open_index(const IndexShape&
shape) {
:
format::IndexConfig::kDocsPositions;
core.stats = shape.stats;
core.section_refs = refs;
+ core.gram_scheme = gram_scheme;
ByteSink core_frame;
const Status core_status = format::encode_core_metadata(core, &core_frame);
EXPECT_TRUE(core_status.ok()) << core_status.to_string();
@@ -248,6 +252,80 @@ TEST(SniiCompactionEligibilityTest,
DestinationWritesNormsExactlyWhenAnalyzed) {
EXPECT_FALSE(keyword.destination_writes_norms);
}
+TEST(SniiCompactionEligibilityTest,
AcceptsLegacyDocsOnlySourcesWithoutPhraseProperty) {
+ auto first = open_index(IndexShape {.tier = format::IndexTier::kT1});
+ auto second = open_index(IndexShape {.tier = format::IndexTier::kT1});
+ auto first_meta = make_index({});
+ auto second_meta = make_index({});
+ auto destination = make_index({});
+ std::vector sources {source(*first, *first_meta), source(*second,
*second_meta)};
+ compaction::SniiCompactionEligibility eligibility;
+
+ const Status status =
+ compaction::validate_snii_compaction_eligibility(sources,
*destination, &eligibility);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ EXPECT_FALSE(eligibility.destination_has_positions);
+ EXPECT_FALSE(eligibility.destination_writes_norms);
+}
+
+TEST(SniiCompactionEligibilityTest, RejectsDocsOnlyGramSources) {
+ auto gram = open_index(IndexShape {.tier = format::IndexTier::kT1},
+ doris::segment_v2::gram::GramScheme {});
+ auto index_meta = make_index({{"support_phrase", "false"}});
+ std::vector sources {source(*gram, *index_meta)};
+ compaction::SniiCompactionEligibility eligibility;
+
+ expect_rejected(
+ compaction::validate_snii_compaction_eligibility(sources,
*index_meta, &eligibility),
+ "gram scheme");
+}
+
+TEST(SniiCompactionEligibilityTest,
AcceptsPositionedSourcesWithoutPhraseProperty) {
+ auto positioned = open_index({});
+ auto source_meta = make_index({});
+ auto destination = make_index({});
+ std::vector sources {source(*positioned, *source_meta)};
+ compaction::SniiCompactionEligibility eligibility;
+
+ const Status status =
+ compaction::validate_snii_compaction_eligibility(sources,
*destination, &eligibility);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ EXPECT_TRUE(eligibility.destination_has_positions);
+}
+
+TEST(SniiCompactionEligibilityTest, RejectsMixedPostingShapes) {
+ auto docs_only = open_index(IndexShape {.tier = format::IndexTier::kT1});
+ auto positioned = open_index({});
+ auto first_meta = make_index({});
+ auto second_meta = make_index({});
+ auto destination = make_index({});
+ std::vector sources {source(*docs_only, *first_meta), source(*positioned,
*second_meta)};
+ compaction::SniiCompactionEligibility eligibility;
+
+ expect_rejected(
+ compaction::validate_snii_compaction_eligibility(sources,
*destination, &eligibility),
+ "position shape");
+}
+
+TEST(SniiCompactionEligibilityTest,
RespectsExplicitPhrasePolicyForDocsOnlySources) {
+ auto docs_only = open_index(IndexShape {.tier = format::IndexTier::kT1});
+ auto no_phrase_meta = make_index({{"support_phrase", "false"}});
+ auto no_phrase_destination = make_index({{"support_phrase", "false"}});
+ std::vector no_phrase_sources {source(*docs_only, *no_phrase_meta)};
+ compaction::SniiCompactionEligibility eligibility;
+ ASSERT_TRUE(compaction::validate_snii_compaction_eligibility(
+ no_phrase_sources, *no_phrase_destination,
&eligibility)
+ .ok());
+ EXPECT_FALSE(eligibility.destination_has_positions);
+
+ auto phrase_meta = make_index({{"support_phrase", "true"}});
+ auto phrase_destination = make_index({{"support_phrase", "true"}});
+ std::vector phrase_sources {source(*docs_only, *phrase_meta)};
+ expect_rejected(compaction::validate_snii_compaction_eligibility(
+ phrase_sources, *phrase_destination, &eligibility),
+ "position shape");
+}
+
// The destination follows the norms policy shared with fresh writes: the
"norms" property, and on
// a variant path inverted_index_skip_norms_for_variant, which wins over the
property.
TEST(SniiCompactionEligibilityTest,
DestinationWritesNormsFollowSharedNormsPolicy) {
diff --git
a/be/test/storage/index/inverted/snii/compaction/snii_index_compaction_test.cpp
b/be/test/storage/index/inverted/snii/compaction/snii_index_compaction_test.cpp
index b7ef12ced4b..594a45c5a5c 100644
---
a/be/test/storage/index/inverted/snii/compaction/snii_index_compaction_test.cpp
+++
b/be/test/storage/index/inverted/snii/compaction/snii_index_compaction_test.cpp
@@ -77,6 +77,13 @@ SniiIndexInput make_input(uint32_t doc_count,
std::vector<uint32_t> null_docids,
return input;
}
+SniiIndexInput make_docs_only_input(uint32_t doc_count, std::vector<uint32_t>
null_docids,
+ std::vector<TermPostings> terms) {
+ SniiIndexInput input = make_input(doc_count, std::move(null_docids),
std::move(terms));
+ input.config = format::IndexConfig::kDocsOnly;
+ return input;
+}
+
// T2 input with norms (A2: analyzed indexes with positions always write
norms). Caller-supplied
// norms must match per-document posting frequencies so compaction can rebuild
identical bytes.
SniiIndexInput make_norms_input(uint32_t doc_count, std::vector<uint32_t>
null_docids,
@@ -398,6 +405,51 @@ TEST(SniiIndexCompactionTest,
MergesDeletesNullsAndInterleavedSourcesByteIdentic
EXPECT_EQ(merged_indexes[1].stats().null_count, 1U);
}
+TEST(SniiIndexCompactionTest, MergesDocsOnlySourcesByteIdenticallyToRebuild) {
+ OpenedIndex source_zero;
+ OpenedIndex source_one;
+ build_index(make_docs_only_input(4, {1}, source_zero_terms()),
&source_zero,
+ reader::LogicalIndexOpenMode::kCompaction);
+ build_index(make_docs_only_input(3, {2}, source_one_terms()), &source_one,
+ reader::LogicalIndexOpenMode::kCompaction);
+
+ const RowIdConversionMap conversion = {
+ {{0, 0}, {0, 2}, {1, 0}, kDeleted},
+ {{0, 1}, {0, 3}, {1, 1}},
+ };
+ const std::vector<uint32_t> destination_rows = {4, 2};
+ auto validated = make_validated_conversion(&conversion, {4, 3},
destination_rows);
+ ASSERT_NE(validated, nullptr);
+
+ compaction::SniiCompactionEligibility eligibility;
+ eligibility.destination_has_positions = false;
+ std::unique_ptr<SniiPlainT2MergePlan> plan;
+ assert_ok(SniiPlainT2MergePlan::prepare({&source_zero.index,
&source_one.index}, *validated,
+ eligibility, 1U << 20, &plan));
+ ASSERT_NE(plan, nullptr);
+ EXPECT_EQ(plan->destination_index_config(),
format::IndexConfig::kDocsOnly);
+
+ std::array<MemoryFile, 2> merged_files;
+ std::array<std::unique_ptr<SniiCompoundWriter>, 2> compounds;
+ std::array<SniiStreamedIndexSession*, 2> sessions = {nullptr, nullptr};
+ for (size_t i = 0; i < compounds.size(); ++i) {
+ compounds[i] = std::make_unique<SniiCompoundWriter>(&merged_files[i]);
+ SniiIndexInput input =
+ make_docs_only_input(destination_rows[i],
plan->destination_null_docids(i), {});
+ assert_ok(compounds[i]->begin_streamed_index(std::move(input),
&sessions[i]));
+ }
+ assert_ok(plan->execute(sessions));
+ for (auto& compound : compounds) {
+ assert_ok(compound->finish());
+ }
+
+ std::array<OpenedIndex, 2> expected;
+ build_index(make_docs_only_input(4, {2}, destination_zero_terms()),
expected.data());
+ build_index(make_docs_only_input(2, {1}, destination_one_terms()),
&expected[1]);
+ expect_identical_index_image(merged_files.data(), &expected[0].file);
+ expect_identical_index_image(&merged_files[1], &expected[1].file);
+}
+
TEST(SniiIndexCompactionTest,
MergesTwentyFourRunSourcesByteIdenticallyToReferenceRebuild) {
constexpr size_t kSourceCount = 24;
constexpr uint32_t kDocsPerSource = 400;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
