SteNicholas commented on code in PR #490:
URL: https://github.com/apache/paimon-cpp/pull/490#discussion_r4453150246
##########
src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp:
##########
@@ -190,4 +192,160 @@ TEST_F(CompleteIndexScoreFileBatchReaderTest,
TestSetReadSchemaReordersScoreColu
ASSERT_TRUE(expected_array->Equals(*result_array)) <<
result_array->ToString();
}
+TEST_F(CompleteIndexScoreFileBatchReaderTest,
TestMultipleScoresFollowSelectedRowsAndProjection) {
+ arrow::FieldVector fields = {arrow::field("score_z", arrow::float32()),
+ arrow::field("f0", arrow::int32()),
+ arrow::field("score_a", arrow::float32())};
+ auto type = arrow::struct_(fields);
+ auto data = arrow::ipc::internal::json::ArrayFromJSON(type, R"([
+ [null, 10, null], [null, 11, null], [null, 12, null],
+ [null, 13, null], [null, 14, null], [null, 15, null],
+ [null, 16, null], [null, 17, null], [null, 18, null],
+ [null, 19, null], [null, 20, null], [null, 21, null]
+ ])")
+ .ValueOrDie();
+ auto selected = RoaringBitmap32::From({0, 2, 8, 9, 10, 11});
+ auto inner = std::make_unique<MockFileBatchReader>(data, type, selected,
+ /*read_batch_size=*/3);
+ inner->EnableRandomizeBatchSize(false);
+ std::map<std::string, std::vector<float>> scores = {
+ {"score_a", {1.25f, 2.5f, 3.75f, 5.0f, 6.25f, 7.5f}},
+ {"score_z", {12.5f, 25.0f, 37.5f, 50.0f, 62.5f, 75.0f}}};
+ CompleteIndexScoreFileBatchReader reader(std::move(inner), scores,
GetArrowPool(pool_));
+
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap first,
reader.NextBatchWithBitmap());
+ ASSERT_EQ(RoaringBitmap32::From({0, 2}), first.second);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> first_array,
+
ReadResultCollector::GetArray(std::move(first.first)));
+ auto expected_first = arrow::ipc::internal::json::ArrayFromJSON(
+ type, R"([[12.5, 10, 1.25], [null, 11, null],
[25, 12, 2.5]])")
+ .ValueOrDie();
+ ASSERT_TRUE(expected_first->Equals(*first_array)) <<
first_array->ToString();
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result_array,
+ ReadResultCollector::CollectResult(&reader));
+ std::shared_ptr<arrow::ChunkedArray> expected_array;
+ ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(
+ type, {R"([[37.5, 18, 3.75], [50, 19, 5], [62.5, 20,
6.25], [75, 21, 7.5]])"},
+ &expected_array)
+ .ok());
+ ASSERT_TRUE(expected_array->Equals(*result_array)) <<
result_array->ToString();
+
+ arrow::FieldVector reordered_fields = {fields[2], fields[0], fields[1]};
+ ::ArrowSchema read_schema;
+ ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(reordered_fields),
&read_schema).ok());
+ ASSERT_OK(reader.SetReadSchema(&read_schema, /*predicate=*/nullptr,
+ /*selection_bitmap=*/selected));
+ ASSERT_OK_AND_ASSIGN(result_array,
ReadResultCollector::CollectResult(&reader));
+ ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(
+ arrow::struct_(reordered_fields),
+ {R"([[1.25, 12.5, 10], [2.5, 25, 12], [3.75, 37.5, 18],
+ [5, 50, 19], [6.25, 62.5, 20], [7.5, 75, 21]])"},
+ &expected_array)
+ .ok());
+ ASSERT_TRUE(expected_array->Equals(*result_array)) <<
result_array->ToString();
+}
+
+TEST_F(CompleteIndexScoreFileBatchReaderTest,
TestMultipleScoresWithEmptyResult) {
+ auto type = arrow::struct_(
+ {arrow::field("score_a", arrow::float32()), arrow::field("score_b",
arrow::float32())});
+ for (const std::string& json : {"[]", "[[null, null]]"}) {
Review Comment:
Please avoid binding `const std::string&` to elements of an initializer list
of string literals here and in the equivalent loops at lines 270 and 290. Each
element is a `const char*`, so the implicit conversion creates a temporary
`std::string` and GCC reports `-Werror=range-loop-construct`. This already
fails the [GCC CI
build](https://github.com/apache/paimon-cpp/actions/runs/41423557423/job/125204618398),
and I reproduced the diagnostic with GCC locally. An explicitly typed string
container or a `const std::string` loop variable fixes the build.
##########
src/paimon/core/operation/file_index_read_executor.cpp:
##########
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/operation/file_index_read_executor.h"
+
+#include <algorithm>
+#include <cmath>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/c/abi.h"
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/reader/complete_index_score_file_batch_reader.h"
+#include "paimon/common/reader/predicate_batch_reader.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/deletionvectors/bitmap64_deletion_vector.h"
+#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/file_index_evaluator.h"
+#include "paimon/core/io/file_index_options.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/file_index/bitmap_index_result.h"
+#include "paimon/file_index/file_index_result.h"
+#include "paimon/file_index/scored_file_index_result.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/full_text_search.h"
+#include "paimon/predicate/function.h"
+#include "paimon/predicate/leaf_predicate.h"
+#include "paimon/predicate/vector_search.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+namespace {
+
+Result<std::string> ResolveSearchFieldName(
+ const std::string& field_name, const std::map<std::string, std::string>&
field_name_mapping,
+ const std::string& file_name) {
+ auto iter = field_name_mapping.find(field_name);
+ if (iter == field_name_mapping.end()) {
+ return Status::Invalid(
+ fmt::format("Search field '{}' has no matching field ID in data
schema for file '{}'",
+ field_name, file_name));
+ }
+ return iter->second;
+}
+
+} // namespace
+
+Result<std::unique_ptr<FileIndexReadExecutor>> FileIndexReadExecutor::Create(
+ const RawFileSplitRead& split_read, const BinaryRow& partition,
+ const std::shared_ptr<DataFileMeta>& file, const FieldMappingBuilder*
field_mapping_builder,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::map<std::string, std::string>& extra_format_options) {
+ std::unique_ptr<FileIndexReadExecutor> executor(new FileIndexReadExecutor(
+ split_read, partition, file, field_mapping_builder, path_factory,
extra_format_options));
+ PAIMON_RETURN_NOT_OK(executor->Prepare());
+ return executor;
+}
+
+FileIndexReadExecutor::FileIndexReadExecutor(
+ const RawFileSplitRead& split_read, const BinaryRow& partition,
+ const std::shared_ptr<DataFileMeta>& file, const FieldMappingBuilder*
field_mapping_builder,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::map<std::string, std::string>& extra_format_options)
+ : split_read_(split_read),
+ partition_(partition),
+ file_(file),
+ field_mapping_builder_(field_mapping_builder),
+ path_factory_(path_factory),
+ extra_format_options_(extra_format_options) {}
+
+Status FileIndexReadExecutor::Prepare() {
+ std::shared_ptr<TableSchema> table_schema =
split_read_.context_->GetTableSchema();
+ if (file_->schema_id != table_schema->Id()) {
+ PAIMON_ASSIGN_OR_RAISE(table_schema,
+
split_read_.schema_manager_->ReadSchema(file_->schema_id));
+ }
+ const auto data_schema =
DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+ if (file_->row_count < 0 || file_->row_count > RoaringBitmap32::MAX_VALUE)
{
+ return Status::NotImplemented(
+ "Precise file-index evaluation requires bitmap32 physical row
positions");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> data_fields,
+
DataField::ConvertArrowSchemaToDataFields(data_schema));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::vector<DataField> read_fields,
+
DataField::ConvertArrowSchemaToDataFields(split_read_.context_->GetReadSchema()));
+ auto field_names = FieldMappingBuilder::CreateFieldNameMapping(
+ split_read_.context_->GetTableSchema()->Fields(), data_fields);
+ std::map<std::string, std::string> index_options;
+ PAIMON_ASSIGN_OR_RAISE(index_options,
+ FileIndexOptions::RewriteFieldNames(
+ split_read_.context_->GetCoreOptions().ToMap(),
field_names));
+ PAIMON_ASSIGN_OR_RAISE(indexes_, FileIndexEvaluator::Create(
+ data_schema, index_options,
path_factory_, file_,
+
split_read_.context_->GetCoreOptions().GetFileSystem(),
+ /*only_use_embedding_index=*/false,
split_read_.pool_));
+
+ std::map<std::string, DataField> read_by_name;
+ for (const auto& field : read_fields) {
+ read_by_name.emplace(field.Name(), field);
+ }
+ std::map<int32_t, std::pair<int32_t, DataField>> data_by_id;
+ for (size_t i = 0; i < data_fields.size(); ++i) {
+ data_by_id.emplace(data_fields[i].Id(), std::make_pair(i,
data_fields[i]));
+ }
+ // Precheck the entire tree before an empty candidate set can
short-circuit it.
+ PAIMON_RETURN_NOT_OK(
+ PreparePredicate(split_read_.context_->GetPredicate(), read_by_name,
data_by_id));
+ return PrepareSearch(field_names);
+}
+
+Status FileIndexReadExecutor::PreparePredicate(
+ const std::shared_ptr<Predicate>& predicate,
+ const std::map<std::string, DataField>& read_by_name,
+ const std::map<int32_t, std::pair<int32_t, DataField>>& data_by_id) {
+ if (!predicate) {
+ return Status::OK();
+ }
+ if (auto compound =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
+ const Function::Type type = compound->GetFunction().GetType();
+ if (type != Function::Type::AND && type != Function::Type::OR) {
+ return Status::Invalid(
+ fmt::format("unsupported compound file predicate {}",
predicate->ToString()));
+ }
+ for (const auto& child : compound->Children()) {
+ PAIMON_RETURN_NOT_OK(PreparePredicate(child, read_by_name,
data_by_id));
+ }
+ return Status::OK();
+ }
+ auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate);
+ if (!leaf) {
+ return Status::Invalid("File predicate must contain leaf or compound
predicates");
+ }
+ bool full_text = leaf->GetFunction().GetType() ==
Function::Type::FULL_TEXT_MATCH;
+ if (!full_text &&
+ ObjectUtils::Contains(split_read_.context_->GetPartitionKeys(),
leaf->FieldName())) {
+ mapped_leaves_.emplace(leaf, nullptr);
+ return Status::OK();
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::optional<std::shared_ptr<Predicate>> mapped,
+ FieldMappingBuilder::ReconstructPredicateWithDataFields(leaf,
read_by_name, data_by_id));
+ std::shared_ptr<LeafPredicate> mapped_leaf;
+ if (mapped) {
+ mapped_leaf = std::dynamic_pointer_cast<LeafPredicate>(*mapped);
+ if (!mapped_leaf) {
+ return Status::Invalid("Mapped file predicate must be a leaf");
+ }
+ if (full_text) {
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::shared_ptr<FileIndexReader> reader,
+ indexes_->GetSearchReader(mapped_leaf->FieldName(),
+
FileIndexReader::SearchType::FULL_TEXT_MATCH));
+ }
+ }
+ mapped_leaves_.emplace(leaf, std::move(mapped_leaf));
+ return Status::OK();
+}
+
+Status FileIndexReadExecutor::PrepareSearch(const std::map<std::string,
std::string>& field_names) {
+ if (split_read_.context_->GetVectorSearch()) {
+ vector_search_ =
std::make_shared<VectorSearch>(*split_read_.context_->GetVectorSearch());
+ PAIMON_ASSIGN_OR_RAISE(
+ vector_search_->field_name,
+ ResolveSearchFieldName(vector_search_->field_name, field_names,
file_->file_name));
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::shared_ptr<FileIndexReader> reader,
+ indexes_->GetSearchReader(vector_search_->field_name,
+
FileIndexReader::SearchType::VECTOR_SEARCH));
+ }
+ if (split_read_.context_->GetFullTextSearch()) {
+ full_text_search_ =
+
std::make_shared<FullTextSearch>(*split_read_.context_->GetFullTextSearch());
+ PAIMON_ASSIGN_OR_RAISE(
+ full_text_search_->field_name,
+ ResolveSearchFieldName(full_text_search_->field_name, field_names,
file_->file_name));
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::shared_ptr<FileIndexReader> reader,
+ indexes_->GetSearchReader(full_text_search_->field_name,
+
FileIndexReader::SearchType::FULL_TEXT_SEARCH));
+ }
+ return Status::OK();
+}
+
+Result<RoaringBitmap32> FileIndexReadExecutor::CreateCandidates(
+ const DeletionVector::Factory& dv_factory,
+ const std::optional<std::vector<Range>>& row_ranges) const {
+ RoaringBitmap32 candidates;
+ if (row_ranges) {
+ for (const auto& range : *row_ranges) {
+ candidates.AddRange(static_cast<int32_t>(range.from),
+ static_cast<int32_t>(range.to + 1));
+ }
+ } else {
+ candidates.AddRange(0, static_cast<int32_t>(file_->row_count));
+ }
+ if (dv_factory) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DeletionVector> dv,
dv_factory(file_->file_name));
+ if (dynamic_cast<Bitmap64DeletionVector*>(dv.get())) {
+ return Status::NotImplemented(
+ "Precise file-index evaluation does not support bitmap64
deletion vectors");
+ }
+ if (auto bitmap = dynamic_cast<BitmapDeletionVector*>(dv.get())) {
+ candidates -= *bitmap->GetBitmap();
+ } else if (dv) {
+ return Status::NotImplemented(
+ "Precise file-index evaluation requires bitmap32 deletion
vectors");
+ }
+ }
+ return candidates;
+}
+
+Result<FileIndexReadExecutor::Evaluation> FileIndexReadExecutor::Evaluate(
+ const std::shared_ptr<Predicate>& predicate, const RoaringBitmap32&
candidates) const {
+ if (!predicate || candidates.IsEmpty()) {
+ return Evaluation{candidates, {}};
+ }
+ if (auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
+ return EvaluateLeaf(leaf, candidates);
+ }
+ if (auto compound =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
+ return EvaluateCompound(compound, candidates);
+ }
+ return Status::Invalid(fmt::format("unsupported file predicate {}",
predicate->ToString()));
+}
+
+Result<FileIndexReadExecutor::Evaluation>
FileIndexReadExecutor::EvaluateCompound(
+ const std::shared_ptr<CompoundPredicate>& predicate, const
RoaringBitmap32& candidates) const {
+ const Function::Type type = predicate->GetFunction().GetType();
+ if (type == Function::Type::OR) {
+ Evaluation result;
+ for (const auto& child : predicate->Children()) {
+ // Every branch must see the same input, including rows matched by
earlier branches.
+ PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, Evaluate(child,
candidates));
+ result.rows |= child_result.rows;
+ result.scores.insert(child_result.scores.begin(),
child_result.scores.end());
+ }
+ return result;
+ }
+ if (type != Function::Type::AND) {
+ return Status::Invalid(
+ fmt::format("unsupported compound file predicate {}",
predicate->ToString()));
+ }
+
+ struct Child {
+ std::shared_ptr<Predicate> predicate;
+ std::optional<double> selectivity;
+ };
+ std::vector<Child> children;
+ children.reserve(predicate->Children().size());
+ for (const auto& child : predicate->Children()) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<double> selectivity,
EstimateSelectivity(child));
+ children.push_back({child, selectivity});
+ }
+ std::stable_sort(children.begin(), children.end(), [](const Child& lhs,
const Child& rhs) {
+ if (!lhs.selectivity) {
+ return false;
+ }
+ return !rhs.selectivity || *lhs.selectivity < *rhs.selectivity;
+ });
+
+ Evaluation result{candidates, {}};
+ for (const auto& child : children) {
+ PAIMON_ASSIGN_OR_RAISE(Evaluation child_result,
Evaluate(child.predicate, result.rows));
+ result.rows = std::move(child_result.rows);
+ result.scores.insert(child_result.scores.begin(),
child_result.scores.end());
+ if (result.rows.IsEmpty()) {
+ break;
+ }
+ }
+ return result;
+}
+
+Result<std::optional<double>> FileIndexReadExecutor::EstimateSelectivity(
+ const std::shared_ptr<Predicate>& predicate) const {
+ if (!predicate) {
+ return std::optional<double>();
+ }
+ if (auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<double> estimate,
EstimateLeafSelectivity(leaf));
+ if (estimate && (!std::isfinite(*estimate) || *estimate < 0.0 ||
*estimate > 1.0)) {
+ return std::optional<double>();
+ }
+ return estimate;
+ }
+ auto compound = std::dynamic_pointer_cast<CompoundPredicate>(predicate);
+ if (!compound) {
+ return Status::Invalid(fmt::format("unsupported file predicate {}",
predicate->ToString()));
+ }
+ const Function::Type type = compound->GetFunction().GetType();
+ if (type != Function::Type::AND && type != Function::Type::OR) {
+ return Status::Invalid(
+ fmt::format("unsupported compound file predicate {}",
predicate->ToString()));
+ }
+ std::optional<double> estimate =
+ type == Function::Type::AND ? std::nullopt : std::make_optional(0.0);
+ // These are scheduling hints. Continue visiting all children to propagate
errors.
+ for (const auto& child : compound->Children()) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<double> child_estimate,
EstimateSelectivity(child));
+ if (type == Function::Type::AND) {
+ if (child_estimate) {
+ if (estimate.has_value()) {
+ estimate = std::min(*estimate, *child_estimate);
+ } else {
+ estimate = *child_estimate;
+ }
+ }
+ } else if (estimate && child_estimate) {
+ estimate = std::min(1.0, *estimate + *child_estimate);
+ } else {
+ estimate = std::nullopt;
+ }
+ }
+ return estimate;
+}
+
+Result<std::optional<double>> FileIndexReadExecutor::EstimateLeafSelectivity(
+ const std::shared_ptr<LeafPredicate>& predicate) const {
+ auto iter = mapped_leaves_.find(predicate);
+ if (iter == mapped_leaves_.end()) {
+ return Status::Invalid("File predicate mapping was not prepared");
+ }
+ if (!iter->second) {
+ return std::optional<double>();
+ }
+ return indexes_->EstimateSelectivity(iter->second);
+}
+
+Result<FileIndexReadExecutor::Evaluation> FileIndexReadExecutor::EvaluateLeaf(
+ const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32&
candidates) const {
+ bool is_full_text = predicate->GetFunction().GetType() ==
Function::Type::FULL_TEXT_MATCH;
+ bool needs_scores =
+ is_full_text &&
+ split_read_.context_->NeedsScores(
+ {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH,
predicate->FieldName()});
+ std::shared_ptr<FileIndexResult> index_result = FileIndexResult::Remain();
+ auto iter = mapped_leaves_.find(predicate);
+ if (iter == mapped_leaves_.end()) {
+ return Status::Invalid("File predicate mapping was not prepared");
+ }
+ if (iter->second) {
+ PAIMON_ASSIGN_OR_RAISE(index_result,
indexes_->EvaluateLeaf(iter->second, needs_scores));
+ }
+ if (!index_result) {
+ return Status::Invalid("file predicate index returned a null result");
+ }
+ Evaluation result{candidates, {}};
+ if (needs_scores) {
+ auto scored =
std::dynamic_pointer_cast<ScoredFileIndexResult>(index_result);
+ if (!scored) {
+ return Status::Invalid(
+ fmt::format("full-text index for field '{}' did not return
requested scores",
+ predicate->FieldName()));
+ }
+ result.rows &= scored->GetRowPositions();
+ result.scores.emplace(predicate->FieldName(), std::move(scored));
+ return result;
+ }
+
+ if (auto bitmap =
std::dynamic_pointer_cast<BitmapIndexResult>(index_result)) {
+ PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* rows,
bitmap->GetBitmap());
+ result.rows &= *rows;
+ } else if (is_full_text) {
+ return Status::Invalid(fmt::format("full-text index for field '{}' did
not return a bitmap",
+ predicate->FieldName()));
+ } else if (std::dynamic_pointer_cast<Skip>(index_result)) {
+ return Evaluation{};
+ } else if (!std::dynamic_pointer_cast<Remain>(index_result)) {
+ return Status::Invalid(
+ fmt::format("invalid index result for predicate {}",
predicate->ToString()));
+ }
+ // TODO(lisizhuo.lsz): Whether to ScanScalar depends on whether the index
returns exact results.
+ if (!is_full_text && !result.rows.IsEmpty()) {
+ // Scalar indexes may contain false positives, even if they return a
bitmap.
+ PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 exact_rows,
ScanScalar(predicate, result.rows));
+ result.rows &= exact_rows;
+ }
+ return result;
+}
+
+Result<std::unique_ptr<FileBatchReader>>
FileIndexReadExecutor::CreateSelectedFileReader(
+ const RoaringBitmap32& selection, const FieldMappingBuilder*
field_mapping_builder) const {
+ PAIMON_ASSIGN_OR_RAISE(std::string format, file_->FileFormat());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReaderBuilder> builder,
+ split_read_.PrepareReaderBuilder(format,
extra_format_options_));
+ return split_read_.CreateFieldMappingReader(path_factory_->ToPath(file_),
file_, partition_,
+ std::move(builder),
field_mapping_builder, nullptr,
+ std::nullopt, path_factory_,
&selection);
+}
+
+Result<RoaringBitmap32> FileIndexReadExecutor::ScanScalar(
+ const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32&
input) const {
+ auto field =
split_read_.context_->GetReadSchema()->GetFieldByName(predicate->FieldName());
+ if (!field) {
+ return Status::Invalid(fmt::format(
+ "Scalar predicate field '{}' is missing from the read schema",
predicate->FieldName()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FieldMappingBuilder> field_mapping_builder,
+ FieldMappingBuilder::Create(arrow::schema({field}),
+ split_read_.context_->GetPartitionKeys(),
nullptr));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> reader,
+ CreateSelectedFileReader(input,
field_mapping_builder.get()));
Review Comment:
Please reject predicates on virtual score columns before this path, or
evaluate them against the computed index scores. A valid projection can contain
`_body_INDEX_SCORE`, but this single-column file reader treats that field as
absent from the physical schema and fills it with null; the real scores are
only attached later by `CompleteIndexScoreFileBatchReader`. Consequently,
`FullTextMatch(body, query) AND _body_INDEX_SCORE > 0.5` silently returns no
rows even when matching scores exceed the threshold, while `IS NULL` can retain
rows incorrectly. Please add regression coverage for both comparison and null
predicates on score columns.
##########
src/paimon/core/operation/file_index_read_executor.cpp:
##########
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/operation/file_index_read_executor.h"
+
+#include <algorithm>
+#include <cmath>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/c/abi.h"
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/reader/complete_index_score_file_batch_reader.h"
+#include "paimon/common/reader/predicate_batch_reader.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/deletionvectors/bitmap64_deletion_vector.h"
+#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/file_index_evaluator.h"
+#include "paimon/core/io/file_index_options.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/file_index/bitmap_index_result.h"
+#include "paimon/file_index/file_index_result.h"
+#include "paimon/file_index/scored_file_index_result.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/full_text_search.h"
+#include "paimon/predicate/function.h"
+#include "paimon/predicate/leaf_predicate.h"
+#include "paimon/predicate/vector_search.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+namespace {
+
+Result<std::string> ResolveSearchFieldName(
+ const std::string& field_name, const std::map<std::string, std::string>&
field_name_mapping,
+ const std::string& file_name) {
+ auto iter = field_name_mapping.find(field_name);
+ if (iter == field_name_mapping.end()) {
+ return Status::Invalid(
+ fmt::format("Search field '{}' has no matching field ID in data
schema for file '{}'",
+ field_name, file_name));
+ }
+ return iter->second;
+}
+
+} // namespace
+
+Result<std::unique_ptr<FileIndexReadExecutor>> FileIndexReadExecutor::Create(
+ const RawFileSplitRead& split_read, const BinaryRow& partition,
+ const std::shared_ptr<DataFileMeta>& file, const FieldMappingBuilder*
field_mapping_builder,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::map<std::string, std::string>& extra_format_options) {
+ std::unique_ptr<FileIndexReadExecutor> executor(new FileIndexReadExecutor(
+ split_read, partition, file, field_mapping_builder, path_factory,
extra_format_options));
+ PAIMON_RETURN_NOT_OK(executor->Prepare());
+ return executor;
+}
+
+FileIndexReadExecutor::FileIndexReadExecutor(
+ const RawFileSplitRead& split_read, const BinaryRow& partition,
+ const std::shared_ptr<DataFileMeta>& file, const FieldMappingBuilder*
field_mapping_builder,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::map<std::string, std::string>& extra_format_options)
+ : split_read_(split_read),
+ partition_(partition),
+ file_(file),
+ field_mapping_builder_(field_mapping_builder),
+ path_factory_(path_factory),
+ extra_format_options_(extra_format_options) {}
+
+Status FileIndexReadExecutor::Prepare() {
+ std::shared_ptr<TableSchema> table_schema =
split_read_.context_->GetTableSchema();
+ if (file_->schema_id != table_schema->Id()) {
+ PAIMON_ASSIGN_OR_RAISE(table_schema,
+
split_read_.schema_manager_->ReadSchema(file_->schema_id));
+ }
+ const auto data_schema =
DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+ if (file_->row_count < 0 || file_->row_count > RoaringBitmap32::MAX_VALUE)
{
+ return Status::NotImplemented(
+ "Precise file-index evaluation requires bitmap32 physical row
positions");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> data_fields,
+
DataField::ConvertArrowSchemaToDataFields(data_schema));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::vector<DataField> read_fields,
+
DataField::ConvertArrowSchemaToDataFields(split_read_.context_->GetReadSchema()));
+ auto field_names = FieldMappingBuilder::CreateFieldNameMapping(
+ split_read_.context_->GetTableSchema()->Fields(), data_fields);
+ std::map<std::string, std::string> index_options;
+ PAIMON_ASSIGN_OR_RAISE(index_options,
+ FileIndexOptions::RewriteFieldNames(
+ split_read_.context_->GetCoreOptions().ToMap(),
field_names));
+ PAIMON_ASSIGN_OR_RAISE(indexes_, FileIndexEvaluator::Create(
+ data_schema, index_options,
path_factory_, file_,
+
split_read_.context_->GetCoreOptions().GetFileSystem(),
+ /*only_use_embedding_index=*/false,
split_read_.pool_));
+
+ std::map<std::string, DataField> read_by_name;
+ for (const auto& field : read_fields) {
+ read_by_name.emplace(field.Name(), field);
+ }
+ std::map<int32_t, std::pair<int32_t, DataField>> data_by_id;
+ for (size_t i = 0; i < data_fields.size(); ++i) {
+ data_by_id.emplace(data_fields[i].Id(), std::make_pair(i,
data_fields[i]));
+ }
+ // Precheck the entire tree before an empty candidate set can
short-circuit it.
+ PAIMON_RETURN_NOT_OK(
+ PreparePredicate(split_read_.context_->GetPredicate(), read_by_name,
data_by_id));
+ return PrepareSearch(field_names);
+}
+
+Status FileIndexReadExecutor::PreparePredicate(
+ const std::shared_ptr<Predicate>& predicate,
+ const std::map<std::string, DataField>& read_by_name,
+ const std::map<int32_t, std::pair<int32_t, DataField>>& data_by_id) {
+ if (!predicate) {
+ return Status::OK();
+ }
+ if (auto compound =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
+ const Function::Type type = compound->GetFunction().GetType();
+ if (type != Function::Type::AND && type != Function::Type::OR) {
+ return Status::Invalid(
+ fmt::format("unsupported compound file predicate {}",
predicate->ToString()));
+ }
+ for (const auto& child : compound->Children()) {
+ PAIMON_RETURN_NOT_OK(PreparePredicate(child, read_by_name,
data_by_id));
+ }
+ return Status::OK();
+ }
+ auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate);
+ if (!leaf) {
+ return Status::Invalid("File predicate must contain leaf or compound
predicates");
+ }
+ bool full_text = leaf->GetFunction().GetType() ==
Function::Type::FULL_TEXT_MATCH;
+ if (!full_text &&
+ ObjectUtils::Contains(split_read_.context_->GetPartitionKeys(),
leaf->FieldName())) {
+ mapped_leaves_.emplace(leaf, nullptr);
+ return Status::OK();
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::optional<std::shared_ptr<Predicate>> mapped,
+ FieldMappingBuilder::ReconstructPredicateWithDataFields(leaf,
read_by_name, data_by_id));
+ std::shared_ptr<LeafPredicate> mapped_leaf;
+ if (mapped) {
+ mapped_leaf = std::dynamic_pointer_cast<LeafPredicate>(*mapped);
+ if (!mapped_leaf) {
+ return Status::Invalid("Mapped file predicate must be a leaf");
+ }
+ if (full_text) {
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::shared_ptr<FileIndexReader> reader,
+ indexes_->GetSearchReader(mapped_leaf->FieldName(),
+
FileIndexReader::SearchType::FULL_TEXT_MATCH));
+ }
+ }
+ mapped_leaves_.emplace(leaf, std::move(mapped_leaf));
+ return Status::OK();
+}
+
+Status FileIndexReadExecutor::PrepareSearch(const std::map<std::string,
std::string>& field_names) {
+ if (split_read_.context_->GetVectorSearch()) {
+ vector_search_ =
std::make_shared<VectorSearch>(*split_read_.context_->GetVectorSearch());
+ PAIMON_ASSIGN_OR_RAISE(
+ vector_search_->field_name,
+ ResolveSearchFieldName(vector_search_->field_name, field_names,
file_->file_name));
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::shared_ptr<FileIndexReader> reader,
+ indexes_->GetSearchReader(vector_search_->field_name,
+
FileIndexReader::SearchType::VECTOR_SEARCH));
+ }
+ if (split_read_.context_->GetFullTextSearch()) {
+ full_text_search_ =
+
std::make_shared<FullTextSearch>(*split_read_.context_->GetFullTextSearch());
+ PAIMON_ASSIGN_OR_RAISE(
+ full_text_search_->field_name,
+ ResolveSearchFieldName(full_text_search_->field_name, field_names,
file_->file_name));
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::shared_ptr<FileIndexReader> reader,
+ indexes_->GetSearchReader(full_text_search_->field_name,
+
FileIndexReader::SearchType::FULL_TEXT_SEARCH));
+ }
+ return Status::OK();
+}
+
+Result<RoaringBitmap32> FileIndexReadExecutor::CreateCandidates(
+ const DeletionVector::Factory& dv_factory,
+ const std::optional<std::vector<Range>>& row_ranges) const {
+ RoaringBitmap32 candidates;
+ if (row_ranges) {
+ for (const auto& range : *row_ranges) {
+ candidates.AddRange(static_cast<int32_t>(range.from),
+ static_cast<int32_t>(range.to + 1));
+ }
+ } else {
+ candidates.AddRange(0, static_cast<int32_t>(file_->row_count));
+ }
+ if (dv_factory) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DeletionVector> dv,
dv_factory(file_->file_name));
+ if (dynamic_cast<Bitmap64DeletionVector*>(dv.get())) {
+ return Status::NotImplemented(
+ "Precise file-index evaluation does not support bitmap64
deletion vectors");
+ }
+ if (auto bitmap = dynamic_cast<BitmapDeletionVector*>(dv.get())) {
+ candidates -= *bitmap->GetBitmap();
+ } else if (dv) {
+ return Status::NotImplemented(
+ "Precise file-index evaluation requires bitmap32 deletion
vectors");
+ }
+ }
+ return candidates;
+}
+
+Result<FileIndexReadExecutor::Evaluation> FileIndexReadExecutor::Evaluate(
+ const std::shared_ptr<Predicate>& predicate, const RoaringBitmap32&
candidates) const {
+ if (!predicate || candidates.IsEmpty()) {
+ return Evaluation{candidates, {}};
+ }
+ if (auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
+ return EvaluateLeaf(leaf, candidates);
+ }
+ if (auto compound =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
+ return EvaluateCompound(compound, candidates);
+ }
+ return Status::Invalid(fmt::format("unsupported file predicate {}",
predicate->ToString()));
+}
+
+Result<FileIndexReadExecutor::Evaluation>
FileIndexReadExecutor::EvaluateCompound(
+ const std::shared_ptr<CompoundPredicate>& predicate, const
RoaringBitmap32& candidates) const {
+ const Function::Type type = predicate->GetFunction().GetType();
+ if (type == Function::Type::OR) {
+ Evaluation result;
+ for (const auto& child : predicate->Children()) {
+ // Every branch must see the same input, including rows matched by
earlier branches.
+ PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, Evaluate(child,
candidates));
+ result.rows |= child_result.rows;
+ result.scores.insert(child_result.scores.begin(),
child_result.scores.end());
+ }
+ return result;
+ }
+ if (type != Function::Type::AND) {
+ return Status::Invalid(
+ fmt::format("unsupported compound file predicate {}",
predicate->ToString()));
+ }
+
+ struct Child {
+ std::shared_ptr<Predicate> predicate;
+ std::optional<double> selectivity;
+ };
+ std::vector<Child> children;
+ children.reserve(predicate->Children().size());
+ for (const auto& child : predicate->Children()) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<double> selectivity,
EstimateSelectivity(child));
+ children.push_back({child, selectivity});
+ }
+ std::stable_sort(children.begin(), children.end(), [](const Child& lhs,
const Child& rhs) {
+ if (!lhs.selectivity) {
+ return false;
+ }
+ return !rhs.selectivity || *lhs.selectivity < *rhs.selectivity;
+ });
+
+ Evaluation result{candidates, {}};
+ for (const auto& child : children) {
+ PAIMON_ASSIGN_OR_RAISE(Evaluation child_result,
Evaluate(child.predicate, result.rows));
+ result.rows = std::move(child_result.rows);
+ result.scores.insert(child_result.scores.begin(),
child_result.scores.end());
+ if (result.rows.IsEmpty()) {
+ break;
+ }
+ }
+ return result;
+}
+
+Result<std::optional<double>> FileIndexReadExecutor::EstimateSelectivity(
+ const std::shared_ptr<Predicate>& predicate) const {
+ if (!predicate) {
+ return std::optional<double>();
+ }
+ if (auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<double> estimate,
EstimateLeafSelectivity(leaf));
+ if (estimate && (!std::isfinite(*estimate) || *estimate < 0.0 ||
*estimate > 1.0)) {
+ return std::optional<double>();
+ }
+ return estimate;
+ }
+ auto compound = std::dynamic_pointer_cast<CompoundPredicate>(predicate);
+ if (!compound) {
+ return Status::Invalid(fmt::format("unsupported file predicate {}",
predicate->ToString()));
+ }
+ const Function::Type type = compound->GetFunction().GetType();
+ if (type != Function::Type::AND && type != Function::Type::OR) {
+ return Status::Invalid(
+ fmt::format("unsupported compound file predicate {}",
predicate->ToString()));
+ }
+ std::optional<double> estimate =
+ type == Function::Type::AND ? std::nullopt : std::make_optional(0.0);
+ // These are scheduling hints. Continue visiting all children to propagate
errors.
+ for (const auto& child : compound->Children()) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<double> child_estimate,
EstimateSelectivity(child));
+ if (type == Function::Type::AND) {
+ if (child_estimate) {
+ if (estimate.has_value()) {
+ estimate = std::min(*estimate, *child_estimate);
+ } else {
+ estimate = *child_estimate;
+ }
+ }
+ } else if (estimate && child_estimate) {
+ estimate = std::min(1.0, *estimate + *child_estimate);
+ } else {
+ estimate = std::nullopt;
+ }
+ }
+ return estimate;
+}
+
+Result<std::optional<double>> FileIndexReadExecutor::EstimateLeafSelectivity(
+ const std::shared_ptr<LeafPredicate>& predicate) const {
+ auto iter = mapped_leaves_.find(predicate);
+ if (iter == mapped_leaves_.end()) {
+ return Status::Invalid("File predicate mapping was not prepared");
+ }
+ if (!iter->second) {
+ return std::optional<double>();
+ }
+ return indexes_->EstimateSelectivity(iter->second);
+}
+
+Result<FileIndexReadExecutor::Evaluation> FileIndexReadExecutor::EvaluateLeaf(
+ const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32&
candidates) const {
+ bool is_full_text = predicate->GetFunction().GetType() ==
Function::Type::FULL_TEXT_MATCH;
+ bool needs_scores =
+ is_full_text &&
+ split_read_.context_->NeedsScores(
+ {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH,
predicate->FieldName()});
+ std::shared_ptr<FileIndexResult> index_result = FileIndexResult::Remain();
+ auto iter = mapped_leaves_.find(predicate);
+ if (iter == mapped_leaves_.end()) {
+ return Status::Invalid("File predicate mapping was not prepared");
+ }
+ if (iter->second) {
+ PAIMON_ASSIGN_OR_RAISE(index_result,
indexes_->EvaluateLeaf(iter->second, needs_scores));
+ }
+ if (!index_result) {
+ return Status::Invalid("file predicate index returned a null result");
+ }
+ Evaluation result{candidates, {}};
+ if (needs_scores) {
+ auto scored =
std::dynamic_pointer_cast<ScoredFileIndexResult>(index_result);
+ if (!scored) {
+ return Status::Invalid(
+ fmt::format("full-text index for field '{}' did not return
requested scores",
+ predicate->FieldName()));
+ }
+ result.rows &= scored->GetRowPositions();
+ result.scores.emplace(predicate->FieldName(), std::move(scored));
+ return result;
+ }
+
+ if (auto bitmap =
std::dynamic_pointer_cast<BitmapIndexResult>(index_result)) {
+ PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* rows,
bitmap->GetBitmap());
+ result.rows &= *rows;
+ } else if (is_full_text) {
+ return Status::Invalid(fmt::format("full-text index for field '{}' did
not return a bitmap",
+ predicate->FieldName()));
+ } else if (std::dynamic_pointer_cast<Skip>(index_result)) {
+ return Evaluation{};
+ } else if (!std::dynamic_pointer_cast<Remain>(index_result)) {
+ return Status::Invalid(
+ fmt::format("invalid index result for predicate {}",
predicate->ToString()));
+ }
+ // TODO(lisizhuo.lsz): Whether to ScanScalar depends on whether the index
returns exact results.
+ if (!is_full_text && !result.rows.IsEmpty()) {
+ // Scalar indexes may contain false positives, even if they return a
bitmap.
+ PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 exact_rows,
ScanScalar(predicate, result.rows));
+ result.rows &= exact_rows;
+ }
+ return result;
+}
+
+Result<std::unique_ptr<FileBatchReader>>
FileIndexReadExecutor::CreateSelectedFileReader(
+ const RoaringBitmap32& selection, const FieldMappingBuilder*
field_mapping_builder) const {
+ PAIMON_ASSIGN_OR_RAISE(std::string format, file_->FileFormat());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReaderBuilder> builder,
+ split_read_.PrepareReaderBuilder(format,
extra_format_options_));
+ return split_read_.CreateFieldMappingReader(path_factory_->ToPath(file_),
file_, partition_,
+ std::move(builder),
field_mapping_builder, nullptr,
+ std::nullopt, path_factory_,
&selection);
+}
+
+Result<RoaringBitmap32> FileIndexReadExecutor::ScanScalar(
+ const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32&
input) const {
+ auto field =
split_read_.context_->GetReadSchema()->GetFieldByName(predicate->FieldName());
+ if (!field) {
+ return Status::Invalid(fmt::format(
+ "Scalar predicate field '{}' is missing from the read schema",
predicate->FieldName()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FieldMappingBuilder> field_mapping_builder,
+ FieldMappingBuilder::Create(arrow::schema({field}),
+ split_read_.context_->GetPartitionKeys(),
nullptr));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> reader,
+ CreateSelectedFileReader(input,
field_mapping_builder.get()));
+ FileBatchReader& file_reader = *reader;
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<PredicateBatchReader> filtered_reader,
+ PredicateBatchReader::Create(std::move(reader), predicate,
+
split_read_.context_->GetArrowMemoryPool()));
+ ScopeGuard close([&]() { filtered_reader->Close(); });
+ RoaringBitmap32 matches;
+ while (true) {
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch,
+ filtered_reader->NextBatchWithBitmap());
+ if (BatchReader::IsEofBatch(batch)) {
+ break;
+ }
Review Comment:
Please release the returned Arrow C array and schema for every non-EOF
batch, including error exits below. This loop only consumes the bitmap and row
positions: it neither imports the batch into Arrow objects nor invokes the C
Data Interface release callbacks. `ReadBatch` uses ordinary
`std::unique_ptr<ArrowArray>` / `std::unique_ptr<ArrowSchema>`, whose
destructors delete only the outer structs, so the exported buffers and schema
allocations leak for every matching batch. Closing the reader does not reclaim
these exported batches. A scope guard around both release callbacks would cover
normal iteration and early returns; a multi-batch allocation/release regression
test would help verify it.
##########
src/paimon/core/operation/abstract_split_read.cpp:
##########
@@ -204,10 +204,12 @@ Result<std::unique_ptr<FileBatchReader>>
AbstractSplitRead::ApplyIndexAndDvReade
std::shared_ptr<FileIndexResult> file_index_result;
if (options_.FileIndexReadEnabled() &&
dynamic_cast<Bitmap64DeletionVector*>(deletion_vector.get()) ==
nullptr) {
- PAIMON_ASSIGN_OR_RAISE(file_index_result,
- FileIndexEvaluator::Evaluate(data_schema,
file_index_options,
- predicate,
data_file_path_factory, file,
-
options_.GetFileSystem(), pool_));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FileIndexEvaluator> evaluator,
+ FileIndexEvaluator::Create(data_schema, file_index_options,
data_file_path_factory,
+ file, options_.GetFileSystem(),
+ /*only_use_embedding_index=*/false,
pool_));
+ PAIMON_ASSIGN_OR_RAISE(file_index_result,
evaluator->Evaluate(predicate));
Review Comment:
Please preserve the no-predicate fast path before creating the evaluator, or
make index loading lazy. Previously `FileIndexEvaluator::Evaluate(nullptr)`
returned before opening an index, but `Create()` now opens and parses the index
container before the null-predicate check in `Evaluate()`. Since file-index
reads are enabled by default, ordinary full scans now access every external
index even with precise evaluation disabled; an unreadable or corrupt index can
also fail a scan that does not need it. Please add a regression test asserting
that a read without a predicate does not open its external index.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
