lxy-9602 commented on code in PR #573:
URL: https://github.com/apache/paimon-cpp/pull/573#discussion_r4593901910


##########
include/paimon/file_index/file_index_result.h:
##########
@@ -43,20 +43,33 @@ class PAIMON_EXPORT FileIndexResult : public 
std::enable_shared_from_this<FileIn
     /// @return Whether the file is remained.
     virtual Result<bool> IsRemain() const = 0;
 
-    /// Compute the intersection of the current result with the provided 
result.
+    /// Whether these rows exactly match the evaluated predicate within its 
candidate scope.

Review Comment:
   Given the current changes, `FileIndexReader` and `FileIndexResult` seem to 
differ from the Java implementation. Please assess whether this will remain 
maintainable in the future.



##########
include/paimon/file_index/scored_file_index_result.h:
##########
@@ -33,14 +33,21 @@ namespace paimon {
 class PAIMON_EXPORT ScoredFileIndexResult : public FileIndexResult {
  public:
     static Result<std::shared_ptr<ScoredFileIndexResult>> 
Create(RoaringBitmap32&& row_positions,
-                                                                 
std::vector<float>&& scores);
+                                                                 
std::vector<float>&& scores,
+                                                                 bool is_exact 
= false);

Review Comment:
   Can scored results be inexact? Or does `scored` not imply ‘exact’?



##########
src/paimon/common/file_index/file_index_reader.cpp:
##########
@@ -21,13 +21,19 @@
 #include <utility>
 
 #include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_utils.h"
 #include "paimon/status.h"
 
 namespace paimon {
 bool FileIndexReader::SupportsSearch(SearchType) const {
     return false;
 }
 
+Result<std::shared_ptr<FileIndexResult>> FileIndexReader::Evaluate(
+    const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32*) {
+    return 
PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(predicate, 
this);
+}
+

Review Comment:
   Return result need intersect with candidate.



##########
src/paimon/core/io/file_index_evaluator.cpp:
##########
@@ -199,22 +203,48 @@ Result<std::shared_ptr<FileIndexResult>> 
FileIndexEvaluator::EvaluateLeaf(
         if (!full_text) {
             return Status::Invalid("FullTextMatch requires a full-text match 
predicate");
         }
-        return reader->VisitFullTextMatch(full_text->GetFullTextMatch(), 
needs_scores);
+        auto match = full_text->GetFullTextMatch();
+        if (candidates) {
+            match = std::make_shared<FullTextMatch>(*match);
+            RoaringBitmap64 eligible(*candidates);
+            if (match->include_row_ids) {
+                eligible &= *match->include_row_ids;
+            }
+            match->include_row_ids = std::move(eligible);
+        }
+        return reader->VisitFullTextMatch(match, needs_scores);
+    }
+    if (candidates && candidates->IsEmpty()) {
+        return FileIndexResult::Skip();
     }
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<FileIndexReader>> 
readers,
                            GetReaders(predicate->FieldName()));
-    std::shared_ptr<FileIndexResult> result = FileIndexResult::Remain();
     for (const auto& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr<FileIndexResult> child,
-            
PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(predicate, 
reader));
-        PAIMON_ASSIGN_OR_RAISE(result, result->And(child));
-        PAIMON_ASSIGN_OR_RAISE(bool remain, result->IsRemain());
-        if (!remain) {
-            break;
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexResult> child,
+                               reader->Evaluate(predicate, candidates));
+        // Inexact indexes may miss matches. Try another index over the 
original scope.
+        // Remain is also inexact and contributes no row selection for this 
predicate.
+        if (!child->IsExact()) {
+            continue;
+        }
+        if (std::dynamic_pointer_cast<Skip>(child)) {
+            return child;
+        }
+        auto bitmap = std::dynamic_pointer_cast<BitmapIndexResult>(child);
+        assert(bitmap);
+        if (!candidates) {
+            return child;
         }
+        return std::make_shared<BitmapIndexResult>(
+            [bitmap, rows = *candidates]() -> Result<RoaringBitmap32> {
+                PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* matches, 
bitmap->GetBitmap());
+                RoaringBitmap32 result(rows);
+                result &= *matches;
+                return result;
+            },
+            /*is_exact=*/true);

Review Comment:
   It seems this place doesn’t assume that the result of `reader->Evaluate` is 
guaranteed to be precisely consistent with the `candidate`—otherwise there 
would be no need to intersect it again.



##########
src/paimon/common/reader/complete_index_score_file_batch_reader.h:
##########


Review Comment:
   Since the production code no longer uses the three-parameter constructor, 
please remove it.



##########
src/paimon/core/io/file_index_evaluator.cpp:
##########
@@ -199,22 +203,48 @@ Result<std::shared_ptr<FileIndexResult>> 
FileIndexEvaluator::EvaluateLeaf(
         if (!full_text) {
             return Status::Invalid("FullTextMatch requires a full-text match 
predicate");
         }
-        return reader->VisitFullTextMatch(full_text->GetFullTextMatch(), 
needs_scores);
+        auto match = full_text->GetFullTextMatch();
+        if (candidates) {
+            match = std::make_shared<FullTextMatch>(*match);
+            RoaringBitmap64 eligible(*candidates);
+            if (match->include_row_ids) {
+                eligible &= *match->include_row_ids;
+            }
+            match->include_row_ids = std::move(eligible);
+        }
+        return reader->VisitFullTextMatch(match, needs_scores);
+    }
+    if (candidates && candidates->IsEmpty()) {
+        return FileIndexResult::Skip();
     }
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<FileIndexReader>> 
readers,
                            GetReaders(predicate->FieldName()));
-    std::shared_ptr<FileIndexResult> result = FileIndexResult::Remain();
     for (const auto& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr<FileIndexResult> child,
-            
PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(predicate, 
reader));
-        PAIMON_ASSIGN_OR_RAISE(result, result->And(child));
-        PAIMON_ASSIGN_OR_RAISE(bool remain, result->IsRemain());
-        if (!remain) {
-            break;
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexResult> child,
+                               reader->Evaluate(predicate, candidates));
+        // Inexact indexes may miss matches. Try another index over the 
original scope.
+        // Remain is also inexact and contributes no row selection for this 
predicate.
+        if (!child->IsExact()) {
+            continue;
+        }
+        if (std::dynamic_pointer_cast<Skip>(child)) {
+            return child;
+        }
+        auto bitmap = std::dynamic_pointer_cast<BitmapIndexResult>(child);
+        assert(bitmap);
+        if (!candidates) {
+            return child;
         }
+        return std::make_shared<BitmapIndexResult>(
+            [bitmap, rows = *candidates]() -> Result<RoaringBitmap32> {
+                PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* matches, 
bitmap->GetBitmap());
+                RoaringBitmap32 result(rows);
+                result &= *matches;
+                return result;
+            },
+            /*is_exact=*/true);
     }
-    return result;
+    return FileIndexResult::Remain();

Review Comment:
   Here I’m a bit curious: if we don’t find a precise match, do we just 
‘Remain’? Instead of returning the bitmap result to enable predicate/bitmap 
pushdown?



##########
src/paimon/core/table/source/key_value_table_read.cpp:
##########
@@ -198,11 +202,19 @@ void KeyValueTableRead::ForceKeepDelete(bool 
force_keep_delete) {
 
 Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
+    if (context_->HasVectorOrFullTextSearch()) {
+        return CreateReader(std::vector<std::shared_ptr<Split>>{split});
+    }
     std::shared_ptr<RealtimeSplit> realtime_split = 
std::dynamic_pointer_cast<RealtimeSplit>(split);
     if (realtime_split) {
         return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
     }
 
+    return CreateDiskReader(split);
+}

Review Comment:
   Why not just call `KeyValueTableRead::CreateReader(
       const std::vector<std::shared_ptr<Split>>& splits)`?



##########
src/paimon/core/table/source/append_only_table_read.cpp:
##########
@@ -141,6 +144,9 @@ AppendOnlyTableRead::AppendOnlyTableRead(const 
std::shared_ptr<FileStorePathFact
 
 Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
+    if (context_->HasVectorOrFullTextSearch()) {
+        return CreateReader(std::vector<std::shared_ptr<Split>>{split});
+    }

Review Comment:
   Just call `AppendOnlyTableRead::CreateReader(
       const std::vector<std::shared_ptr<Split>>& splits)`?



##########
src/paimon/core/table/source/topk_batch_reader.cpp:
##########
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/topk_batch_reader.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/concatenate.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+
+namespace paimon {
+
+TopKBatchReader::TopKBatchReader(std::unique_ptr<BatchReader> reader,
+                                 const std::string& score_field_name, 
ScoreOrder score_order,
+                                 int32_t limit, int32_t batch_size,
+                                 const std::shared_ptr<arrow::Schema>& 
read_schema,
+                                 const std::shared_ptr<arrow::MemoryPool>& 
pool)
+    : pool_(pool),
+      reader_(std::move(reader)),
+      score_name_(score_field_name),
+      score_order_(score_order),
+      limit_(limit),
+      batch_size_(batch_size),
+      read_type_(arrow::struct_(read_schema->fields())),
+      remove_score_(read_schema->GetFieldIndex(score_field_name) < 0) {}
+
+Result<BatchReader::ReadBatch> TopKBatchReader::NextBatch() {
+    if (!initialized_) {
+        initialized_ = true;
+        PAIMON_RETURN_NOT_OK(ReadTopK());
+    }
+    if (!output_ || position_ == output_->length()) {
+        return MakeEofBatch();
+    }
+    return ReadOutput();
+}
+
+std::shared_ptr<Metrics> TopKBatchReader::GetReaderMetrics() const {
+    return reader_->GetReaderMetrics();
+}
+
+void TopKBatchReader::Close() {
+    reader_->Close();
+    output_.reset();
+}
+
+Status TopKBatchReader::CompactCandidates(std::vector<Candidate>* candidates) 
const {

Review Comment:
   Consider use `Concatenate` rather than `Compact`?



##########
src/paimon/core/table/source/topk_batch_reader.cpp:
##########
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/topk_batch_reader.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/concatenate.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+
+namespace paimon {
+
+TopKBatchReader::TopKBatchReader(std::unique_ptr<BatchReader> reader,
+                                 const std::string& score_field_name, 
ScoreOrder score_order,
+                                 int32_t limit, int32_t batch_size,
+                                 const std::shared_ptr<arrow::Schema>& 
read_schema,
+                                 const std::shared_ptr<arrow::MemoryPool>& 
pool)
+    : pool_(pool),
+      reader_(std::move(reader)),
+      score_name_(score_field_name),
+      score_order_(score_order),
+      limit_(limit),
+      batch_size_(batch_size),
+      read_type_(arrow::struct_(read_schema->fields())),
+      remove_score_(read_schema->GetFieldIndex(score_field_name) < 0) {}
+
+Result<BatchReader::ReadBatch> TopKBatchReader::NextBatch() {
+    if (!initialized_) {
+        initialized_ = true;
+        PAIMON_RETURN_NOT_OK(ReadTopK());
+    }
+    if (!output_ || position_ == output_->length()) {
+        return MakeEofBatch();
+    }
+    return ReadOutput();
+}
+
+std::shared_ptr<Metrics> TopKBatchReader::GetReaderMetrics() const {
+    return reader_->GetReaderMetrics();
+}
+
+void TopKBatchReader::Close() {
+    reader_->Close();
+    output_.reset();
+}
+
+Status TopKBatchReader::CompactCandidates(std::vector<Candidate>* candidates) 
const {
+    if (candidates->empty()) {
+        return Status::OK();
+    }
+    arrow::ArrayVector rows;
+    rows.reserve(candidates->size());
+    for (const auto& candidate : *candidates) {
+        rows.push_back(candidate.values->Slice(candidate.row, 1));
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> compact,
+                                      arrow::Concatenate(rows, pool_.get()));
+    for (size_t i = 0; i < candidates->size(); ++i) {
+        (*candidates)[i].values = compact;
+        (*candidates)[i].row = static_cast<int64_t>(i);
+    }
+    return Status::OK();
+}
+
+Status TopKBatchReader::ReadTopK() {
+    std::vector<Candidate> candidates;
+    uint64_t ordinal = 0;
+    bool ascending = score_order_ == ScoreOrder::ASCENDING;
+    auto better = [ascending](const Candidate& left, const Candidate& right) {
+        int32_t comparison = 
FieldsComparator::CompareFloatingPoint(left.score, right.score);
+        if (comparison == 0) {
+            return left.ordinal < right.ordinal;
+        }
+        return ascending ? comparison < 0 : comparison > 0;
+    };
+    while (true) {
+        // NextBatch applies all row-selection bitmaps, including deletion 
vectors.
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());

Review Comment:
   Why is this calling `NextBatch` rather than `NextBatchWith`? In the current 
mode, data is already copied when calling `NextBatch`.



##########
src/paimon/core/table/source/topk_batch_reader.cpp:
##########
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/topk_batch_reader.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/concatenate.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+
+namespace paimon {
+
+TopKBatchReader::TopKBatchReader(std::unique_ptr<BatchReader> reader,
+                                 const std::string& score_field_name, 
ScoreOrder score_order,
+                                 int32_t limit, int32_t batch_size,
+                                 const std::shared_ptr<arrow::Schema>& 
read_schema,
+                                 const std::shared_ptr<arrow::MemoryPool>& 
pool)
+    : pool_(pool),
+      reader_(std::move(reader)),
+      score_name_(score_field_name),
+      score_order_(score_order),
+      limit_(limit),
+      batch_size_(batch_size),
+      read_type_(arrow::struct_(read_schema->fields())),
+      remove_score_(read_schema->GetFieldIndex(score_field_name) < 0) {}
+
+Result<BatchReader::ReadBatch> TopKBatchReader::NextBatch() {
+    if (!initialized_) {
+        initialized_ = true;
+        PAIMON_RETURN_NOT_OK(ReadTopK());
+    }
+    if (!output_ || position_ == output_->length()) {
+        return MakeEofBatch();
+    }
+    return ReadOutput();
+}
+
+std::shared_ptr<Metrics> TopKBatchReader::GetReaderMetrics() const {
+    return reader_->GetReaderMetrics();
+}
+
+void TopKBatchReader::Close() {
+    reader_->Close();
+    output_.reset();
+}
+
+Status TopKBatchReader::CompactCandidates(std::vector<Candidate>* candidates) 
const {
+    if (candidates->empty()) {
+        return Status::OK();
+    }
+    arrow::ArrayVector rows;
+    rows.reserve(candidates->size());
+    for (const auto& candidate : *candidates) {
+        rows.push_back(candidate.values->Slice(candidate.row, 1));
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> compact,
+                                      arrow::Concatenate(rows, pool_.get()));
+    for (size_t i = 0; i < candidates->size(); ++i) {
+        (*candidates)[i].values = compact;
+        (*candidates)[i].row = static_cast<int64_t>(i);
+    }
+    return Status::OK();
+}
+
+Status TopKBatchReader::ReadTopK() {
+    std::vector<Candidate> candidates;
+    uint64_t ordinal = 0;
+    bool ascending = score_order_ == ScoreOrder::ASCENDING;
+    auto better = [ascending](const Candidate& left, const Candidate& right) {
+        int32_t comparison = 
FieldsComparator::CompareFloatingPoint(left.score, right.score);
+        if (comparison == 0) {
+            return left.ordinal < right.ordinal;
+        }
+        return ascending ? comparison < 0 : comparison > 0;
+    };
+    while (true) {
+        // NextBatch applies all row-selection bitmaps, including deletion 
vectors.
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
+        if (IsEofBatch(batch)) {
+            break;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (array->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("TopK reader requires a StructArray");
+        }
+        if (array->length() == 0) {
+            continue;
+        }
+        auto values = checked_pointer_cast<arrow::StructArray>(array);
+        PAIMON_ASSIGN_OR_RAISE(values, 
ArrowUtils::FlattenUnresolvableDictionaries(
+                                           values, read_type_, pool_.get(),
+                                           
/*preserve_layout_recoverable_dictionaries=*/false));
+        auto scores =
+            
std::dynamic_pointer_cast<arrow::FloatArray>(values->GetFieldByName(score_name_));
+        if (!scores || scores->null_count() != 0) {
+            return Status::Invalid("TopK reader requires non-null float search 
scores");
+        }
+        for (int64_t row = 0; row < values->length(); ++row) {
+            const float score = scores->Value(row);
+            Candidate candidate{score, ordinal++, values, row};
+            if (candidates.size() < static_cast<size_t>(limit_)) {
+                candidates.push_back(std::move(candidate));
+                std::push_heap(candidates.begin(), candidates.end(), better);
+            } else if (better(candidate, candidates.front())) {
+                std::pop_heap(candidates.begin(), candidates.end(), better);
+                candidates.back() = std::move(candidate);
+                std::push_heap(candidates.begin(), candidates.end(), better);
+            }
+        }
+        PAIMON_RETURN_NOT_OK(CompactCandidates(&candidates));

Review Comment:
   Why not just use a heap to do the sort?



##########
src/paimon/core/table/source/topk_batch_reader.cpp:
##########
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/topk_batch_reader.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/concatenate.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+
+namespace paimon {
+
+TopKBatchReader::TopKBatchReader(std::unique_ptr<BatchReader> reader,
+                                 const std::string& score_field_name, 
ScoreOrder score_order,
+                                 int32_t limit, int32_t batch_size,
+                                 const std::shared_ptr<arrow::Schema>& 
read_schema,
+                                 const std::shared_ptr<arrow::MemoryPool>& 
pool)
+    : pool_(pool),
+      reader_(std::move(reader)),
+      score_name_(score_field_name),
+      score_order_(score_order),
+      limit_(limit),
+      batch_size_(batch_size),
+      read_type_(arrow::struct_(read_schema->fields())),
+      remove_score_(read_schema->GetFieldIndex(score_field_name) < 0) {}
+
+Result<BatchReader::ReadBatch> TopKBatchReader::NextBatch() {
+    if (!initialized_) {
+        initialized_ = true;
+        PAIMON_RETURN_NOT_OK(ReadTopK());
+    }
+    if (!output_ || position_ == output_->length()) {
+        return MakeEofBatch();
+    }
+    return ReadOutput();
+}
+
+std::shared_ptr<Metrics> TopKBatchReader::GetReaderMetrics() const {
+    return reader_->GetReaderMetrics();
+}
+
+void TopKBatchReader::Close() {
+    reader_->Close();
+    output_.reset();
+}
+
+Status TopKBatchReader::CompactCandidates(std::vector<Candidate>* candidates) 
const {
+    if (candidates->empty()) {
+        return Status::OK();
+    }
+    arrow::ArrayVector rows;
+    rows.reserve(candidates->size());
+    for (const auto& candidate : *candidates) {
+        rows.push_back(candidate.values->Slice(candidate.row, 1));
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> compact,
+                                      arrow::Concatenate(rows, pool_.get()));
+    for (size_t i = 0; i < candidates->size(); ++i) {
+        (*candidates)[i].values = compact;
+        (*candidates)[i].row = static_cast<int64_t>(i);
+    }
+    return Status::OK();
+}
+
+Status TopKBatchReader::ReadTopK() {
+    std::vector<Candidate> candidates;
+    uint64_t ordinal = 0;
+    bool ascending = score_order_ == ScoreOrder::ASCENDING;
+    auto better = [ascending](const Candidate& left, const Candidate& right) {
+        int32_t comparison = 
FieldsComparator::CompareFloatingPoint(left.score, right.score);
+        if (comparison == 0) {
+            return left.ordinal < right.ordinal;
+        }
+        return ascending ? comparison < 0 : comparison > 0;
+    };
+    while (true) {
+        // NextBatch applies all row-selection bitmaps, including deletion 
vectors.
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
+        if (IsEofBatch(batch)) {
+            break;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (array->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("TopK reader requires a StructArray");
+        }
+        if (array->length() == 0) {
+            continue;
+        }
+        auto values = checked_pointer_cast<arrow::StructArray>(array);
+        PAIMON_ASSIGN_OR_RAISE(values, 
ArrowUtils::FlattenUnresolvableDictionaries(
+                                           values, read_type_, pool_.get(),
+                                           
/*preserve_layout_recoverable_dictionaries=*/false));
+        auto scores =
+            
std::dynamic_pointer_cast<arrow::FloatArray>(values->GetFieldByName(score_name_));
+        if (!scores || scores->null_count() != 0) {
+            return Status::Invalid("TopK reader requires non-null float search 
scores");
+        }
+        for (int64_t row = 0; row < values->length(); ++row) {
+            const float score = scores->Value(row);
+            Candidate candidate{score, ordinal++, values, row};
+            if (candidates.size() < static_cast<size_t>(limit_)) {
+                candidates.push_back(std::move(candidate));
+                std::push_heap(candidates.begin(), candidates.end(), better);
+            } else if (better(candidate, candidates.front())) {
+                std::pop_heap(candidates.begin(), candidates.end(), better);
+                candidates.back() = std::move(candidate);
+                std::push_heap(candidates.begin(), candidates.end(), better);
+            }
+        }
+        PAIMON_RETURN_NOT_OK(CompactCandidates(&candidates));

Review Comment:
   Does concatenating multiple batches happen multiple times? At least, if the 
data hasn’t changed, we shouldn’t concatenate again.



##########
src/paimon/core/table/source/key_value_table_read.cpp:
##########
@@ -293,7 +305,24 @@ Result<std::unique_ptr<BatchReader>> 
KeyValueTableRead::CreateReader(
                 
realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
-    return std::make_unique<ConcatBatchReader>(std::move(readers), 
context_->GetArrowMemoryPool());
+    std::unique_ptr<BatchReader> reader =
+        std::make_unique<ConcatBatchReader>(std::move(readers), 
context_->GetArrowMemoryPool());
+    if (!context_->HasVectorOrFullTextSearch()) {
+        return reader;
+    }
+    const auto& vector_search = context_->GetVectorSearch();
+    const auto& full_text_search = context_->GetFullTextSearch();
+    const std::string score_name = SpecialFields::IndexScoreFieldName(
+        vector_search ? vector_search->field_name : 
full_text_search->field_name);
+    const int32_t limit = vector_search ? vector_search->limit : 
full_text_search->limit;
+    const TopKBatchReader::ScoreOrder score_order =
+        vector_search && *vector_search->distance_type != 
VectorSearch::DistanceType::INNER_PRODUCT
+            ? TopKBatchReader::ScoreOrder::ASCENDING
+            : TopKBatchReader::ScoreOrder::DESCENDING;
+    return std::make_unique<TopKBatchReader>(std::move(reader), score_name, 
score_order, limit,
+                                             
context_->GetCoreOptions().GetReadBatchSize(),
+                                             context_->GetReadSchema(),
+                                             context_->GetArrowMemoryPool());

Review Comment:
   Please make sure this part is extracted into a function in the base class. 
The score-sorting logic (ascending/descending) must not be implemented 
separately in both append and the PK table.



##########
src/paimon/core/table/source/key_value_table_read.cpp:
##########
@@ -293,7 +305,24 @@ Result<std::unique_ptr<BatchReader>> 
KeyValueTableRead::CreateReader(
                 
realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
-    return std::make_unique<ConcatBatchReader>(std::move(readers), 
context_->GetArrowMemoryPool());
+    std::unique_ptr<BatchReader> reader =
+        std::make_unique<ConcatBatchReader>(std::move(readers), 
context_->GetArrowMemoryPool());
+    if (!context_->HasVectorOrFullTextSearch()) {
+        return reader;
+    }
+    const auto& vector_search = context_->GetVectorSearch();
+    const auto& full_text_search = context_->GetFullTextSearch();
+    const std::string score_name = SpecialFields::IndexScoreFieldName(
+        vector_search ? vector_search->field_name : 
full_text_search->field_name);
+    const int32_t limit = vector_search ? vector_search->limit : 
full_text_search->limit;
+    const TopKBatchReader::ScoreOrder score_order =
+        vector_search && *vector_search->distance_type != 
VectorSearch::DistanceType::INNER_PRODUCT
+            ? TopKBatchReader::ScoreOrder::ASCENDING
+            : TopKBatchReader::ScoreOrder::DESCENDING;
+    return std::make_unique<TopKBatchReader>(std::move(reader), score_name, 
score_order, limit,
+                                             
context_->GetCoreOptions().GetReadBatchSize(),
+                                             context_->GetReadSchema(),
+                                             context_->GetArrowMemoryPool());

Review Comment:
   May be `ApplyTopkReaderIfNeeded`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to