This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new d21122ec feat(predicate): add full-text match and score projection
semantics (#451)
d21122ec is described below
commit d21122ecb44f8bc0ffa1eea33a018179ba5b993a
Author: lszskye <[email protected]>
AuthorDate: Fri Oct 23 09:44:43 2026 +0800
feat(predicate): add full-text match and score projection semantics (#451)
---
docs/source/api/file_index.rst | 67 +++++
include/paimon/file_index/file_index_reader.h | 5 +
include/paimon/predicate/full_text_match.h | 51 ++++
include/paimon/predicate/function.h | 3 +-
include/paimon/predicate/function_visitor.h | 10 +
include/paimon/predicate/leaf_predicate.h | 3 +-
include/paimon/predicate/predicate_builder.h | 10 +
include/paimon/predicate/predicate_utils.h | 11 +
include/paimon/read_context.h | 34 ++-
src/paimon/CMakeLists.txt | 2 +
src/paimon/common/file_index/file_index_reader.cpp | 5 +
.../common/file_index/file_index_reader_test.cpp | 38 +++
.../common/predicate/full_text_match_function.h | 67 +++++
.../common/predicate/full_text_match_predicate.cpp | 64 ++++
.../common/predicate/full_text_match_predicate.h | 53 ++++
.../predicate/full_text_match_predicate_test.cpp | 148 +++++++++
src/paimon/common/predicate/predicate_builder.cpp | 16 +
src/paimon/common/predicate/predicate_utils.cpp | 32 ++
.../common/predicate/predicate_utils_test.cpp | 43 +++
.../common/predicate/predicate_validator_test.cpp | 43 +++
src/paimon/common/table/special_fields.h | 18 +-
src/paimon/common/table/special_fields_test.cpp | 13 +
src/paimon/core/operation/file_store_scan.cpp | 3 +
src/paimon/core/operation/file_store_scan_test.cpp | 24 ++
.../core/operation/internal_read_context.cpp | 101 ++++++-
src/paimon/core/operation/internal_read_context.h | 31 +-
.../core/operation/internal_read_context_test.cpp | 329 +++++++++++++++++++++
src/paimon/core/operation/read_context.cpp | 18 ++
src/paimon/core/operation/read_context_test.cpp | 37 +++
src/paimon/core/table/source/table_read_test.cpp | 52 ++++
src/paimon/core/table/table.cpp | 14 +
src/paimon/core/utils/field_mapping.cpp | 26 +-
src/paimon/core/utils/field_mapping_test.cpp | 43 +++
test/inte/predicate_abi_inte_test.cpp | 21 ++
34 files changed, 1407 insertions(+), 28 deletions(-)
diff --git a/docs/source/api/file_index.rst b/docs/source/api/file_index.rst
index e95cd26f..b6a0eb14 100644
--- a/docs/source/api/file_index.rst
+++ b/docs/source/api/file_index.rst
@@ -21,9 +21,76 @@ File Index
.. _cpp-api-file-index:
+Full-text predicates and root TopK
+=================================
+
+``PredicateBuilder::FullTextMatch`` creates a ``LeafPredicate`` for a full-text
+condition without TopK truncation. It holds a shared ``FullTextMatch`` request,
+whose query uses the same JSON DSL as ``FullTextSearch``. Do not modify a
request
+after passing it to a predicate or read context.
+
+Compose full-text and scalar predicates with the existing
+``PredicateBuilder::And`` and ``PredicateBuilder::Or`` factories. Every field
+referenced by this tree must appear in the read schema; field indices refer to
+positions in that schema. For example, the following constructs
+``body matches "apple" OR price < 10``:
+
+.. code-block:: cpp
+
+ auto request = std::make_shared<paimon::FullTextMatch>(
+ "body", R"({"match":{"query":"apple"}})");
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<paimon::Predicate> text,
+ paimon::PredicateBuilder::FullTextMatch(0, request));
+ auto price = paimon::PredicateBuilder::LessThan(
+ 1, "price", paimon::FieldType::INT, paimon::Literal(10));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<paimon::Predicate> filter,
+ paimon::PredicateBuilder::Or({text, price}));
+ paimon::ReadContextBuilder builder(table_path);
+ builder.SetReadFieldNames({"body", "price"}).SetPredicate(filter);
+
+``SetVectorSearch`` and ``SetFullTextSearch`` configure a separate root TopK
+request. They are mutually exclusive and require a positive limit. The intended
+semantics are to filter by the complete predicate before ranking; the TopK
search
+field does not need to be projected. A full-text match inside a predicate never
+has a TopK limit.
+
+Score output is requested through an explicit ``_<field>_INDEX_SCORE``
+projection. For example, request the score of a full-text
+predicate by projecting ``_body_INDEX_SCORE``:
+
+.. code-block:: cpp
+
+ auto request = std::make_shared<paimon::FullTextMatch>(
+ "body", R"({"match":{"query":"apple"}})");
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<paimon::Predicate> text,
+ paimon::PredicateBuilder::FullTextMatch(0, request));
+ paimon::ReadContextBuilder builder(table_path);
+ builder.SetReadFieldNames({"body", "_body_INDEX_SCORE"}).SetPredicate(text);
+
+Score fields have type ``float32`` and preserve the source field's exact name,
+including case. Select them with ``SetReadFieldNames`` or ``SetReadSchema``;
+they have no persistent field IDs for ``SetReadFieldIds``. Reading all table
+fields does not implicitly add score fields.
+
+Each projected score must have exactly one search condition as its source.
+The source may be a root TopK request or a full-text predicate. A full-text
+predicate whose score is projected cannot have an OR ancestor, and its field
+cannot occur in another search condition, including the root TopK request.
+An ordinary scalar predicate on the same field is allowed. For example, when
+projecting the score of ``match``, ``match AND (p OR q)`` is valid, while
+``match OR p`` is rejected. Without that score projection, both OR branches and
+repeated full-text search fields are allowed. These checks use the effective
+read projection, after resolving the read schema.
+
Interface
=========
+.. doxygenstruct:: paimon::FullTextMatch
+ :members:
+
.. doxygenclass:: paimon::FileIndexer
:members:
:undoc-members:
diff --git a/include/paimon/file_index/file_index_reader.h
b/include/paimon/file_index/file_index_reader.h
index e6cb39c7..94f758ea 100644
--- a/include/paimon/file_index/file_index_reader.h
+++ b/include/paimon/file_index/file_index_reader.h
@@ -31,6 +31,7 @@
#include "paimon/visibility.h"
namespace paimon {
+struct FullTextMatch;
/// Evaluates filter predicates against a file-level index to determine file
eligibility.
///
/// `FileIndexReader` implements the `FunctionVisitor` interface specialized
to produce
@@ -79,6 +80,10 @@ class PAIMON_EXPORT FileIndexReader : public
FunctionVisitor<std::shared_ptr<Fil
/// Execute full-text search and return matching file-local physical row
positions.
virtual Result<std::shared_ptr<FileIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search);
+
+ /// Return every matching file-local physical row position, without TopK
truncation.
+ Result<std::shared_ptr<FileIndexResult>> VisitFullTextMatch(
+ const std::shared_ptr<FullTextMatch>& full_text_match) override;
};
} // namespace paimon
diff --git a/include/paimon/predicate/full_text_match.h
b/include/paimon/predicate/full_text_match.h
new file mode 100644
index 00000000..51c37536
--- /dev/null
+++ b/include/paimon/predicate/full_text_match.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "paimon/utils/roaring_bitmap64.h"
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+/// A full-text condition returning every matching row without TopK truncation.
+/// Project `_<field>_INDEX_SCORE` in the read schema to return its scores.
+struct PAIMON_EXPORT FullTextMatch {
+ FullTextMatch(const std::string& _field_name, const std::string& _query,
+ std::optional<RoaringBitmap64> _include_row_ids =
std::nullopt)
+ : field_name(_field_name), query(_query),
include_row_ids(std::move(_include_row_ids)) {}
+
+ bool operator==(const FullTextMatch& other) const {
+ return field_name == other.field_name && query == other.query &&
+ include_row_ids == other.include_row_ids;
+ }
+
+ /// Table column containing the indexed text.
+ std::string field_name;
+ /// JSON DSL query with the same syntax and analyzer semantics as
`FullTextSearch::query`.
+ std::string query;
+ /// Optional eligible row ids. File readers use file-local physical row
positions;
+ /// An empty set matches no rows. If not set, all rows are searched.
+ std::optional<RoaringBitmap64> include_row_ids;
+};
+
+} // namespace paimon
diff --git a/include/paimon/predicate/function.h
b/include/paimon/predicate/function.h
index f383f743..5d84f5c1 100644
--- a/include/paimon/predicate/function.h
+++ b/include/paimon/predicate/function.h
@@ -43,7 +43,8 @@ class PAIMON_EXPORT Function {
STARTS_WITH = 13,
ENDS_WITH = 14,
CONTAINS = 15,
- LIKE = 16
+ LIKE = 16,
+ FULL_TEXT_MATCH = 17
};
virtual ~Function() = default;
virtual Type GetType() const = 0;
diff --git a/include/paimon/predicate/function_visitor.h
b/include/paimon/predicate/function_visitor.h
index 8b759ff2..5e8b54b3 100644
--- a/include/paimon/predicate/function_visitor.h
+++ b/include/paimon/predicate/function_visitor.h
@@ -18,15 +18,19 @@
#pragma once
+#include <memory>
#include <string>
#include <vector>
#include "paimon/predicate/leaf_predicate.h"
#include "paimon/predicate/literal.h"
#include "paimon/result.h"
+#include "paimon/status.h"
#include "paimon/visibility.h"
namespace paimon {
+struct FullTextMatch;
+
/// A visitor interface for evaluating filter predicates on indexed columns.
/// @tparam T The result type produced by each visit method (e.g., a file
index result or global
/// index result).
@@ -76,5 +80,11 @@ class PAIMON_EXPORT FunctionVisitor {
/// Evaluates whether string values like the given string.
virtual Result<T> VisitLike(const Literal& literal) = 0;
+
+ /// Evaluates a full-text condition without TopK truncation.
+ /// Visitors without full-text support return NotImplemented.
+ virtual Result<T> VisitFullTextMatch(const std::shared_ptr<FullTextMatch>&
full_text_match) {
+ return Status::NotImplemented("This visitor does not support complete
full-text matching");
+ }
};
} // namespace paimon
diff --git a/include/paimon/predicate/leaf_predicate.h
b/include/paimon/predicate/leaf_predicate.h
index 29698487..b57d8173 100644
--- a/include/paimon/predicate/leaf_predicate.h
+++ b/include/paimon/predicate/leaf_predicate.h
@@ -31,7 +31,8 @@ class LeafFunction;
class Function;
enum class FieldType;
-/// Leaf node of a `Predicate` tree. Compares a field with literals.
+/// Leaf node of a `Predicate` tree describing a condition on one field.
+/// Scalar leaves compare the field with literals; full-text leaves hold a
search request.
class PAIMON_EXPORT LeafPredicate : virtual public Predicate {
public:
int32_t FieldIndex() const {
diff --git a/include/paimon/predicate/predicate_builder.h
b/include/paimon/predicate/predicate_builder.h
index c8cdf13d..4c829408 100644
--- a/include/paimon/predicate/predicate_builder.h
+++ b/include/paimon/predicate/predicate_builder.h
@@ -27,6 +27,7 @@
#include "paimon/visibility.h"
namespace paimon {
+struct FullTextMatch;
class Literal;
enum class FieldType;
@@ -39,6 +40,15 @@ class PAIMON_EXPORT PredicateBuilder {
PredicateBuilder() = delete;
~PredicateBuilder() = delete;
+ /// Create a full-text predicate returning all matching rows without TopK
truncation.
+ /// The field must have STRING type and a full-text index.
+ /// Project `_<field>_INDEX_SCORE` in the read schema to request score
output.
+ /// @param field_index The index of the field in read schema (0-based).
+ /// @param match The full-text request, including the field name, JSON DSL
query and
+ /// optional eligible row ids.
+ static Result<std::shared_ptr<Predicate>> FullTextMatch(
+ int32_t field_index, const std::shared_ptr<FullTextMatch>& match);
+
/// Create an equality predicate (field == literal).
///
/// @param field_index The index of the field in read schema (0-based).
diff --git a/include/paimon/predicate/predicate_utils.h
b/include/paimon/predicate/predicate_utils.h
index cef3b5fa..24cc47c0 100644
--- a/include/paimon/predicate/predicate_utils.h
+++ b/include/paimon/predicate/predicate_utils.h
@@ -61,6 +61,9 @@ class PAIMON_EXPORT PredicateUtils {
static Status GetAllNames(const std::shared_ptr<Predicate>& predicate,
std::set<std::string>* field_names);
+ /// Whether the predicate contains a full-text matching condition.
+ static bool ContainsFullTextMatch(const std::shared_ptr<Predicate>&
predicate);
+
template <typename T>
static Result<T> VisitPredicate(const std::shared_ptr<LeafPredicate>&
predicate,
const std::shared_ptr<FunctionVisitor<T>>&
visitor) {
@@ -114,6 +117,11 @@ class PAIMON_EXPORT PredicateUtils {
assert(predicate->Literals().size() == 1);
return visitor->VisitLike(predicate->Literals()[0]);
}
+ case Function::Type::FULL_TEXT_MATCH: {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FullTextMatch>
full_text_match,
+ GetFullTextMatch(predicate));
+ return visitor->VisitFullTextMatch(full_text_match);
+ }
default:
return Status::Invalid("invalid " +
predicate->GetFunction().ToString() +
" function in leaf predicate");
@@ -121,6 +129,9 @@ class PAIMON_EXPORT PredicateUtils {
}
private:
+ static Result<std::shared_ptr<FullTextMatch>> GetFullTextMatch(
+ const std::shared_ptr<LeafPredicate>& predicate);
+
static Result<std::optional<std::shared_ptr<Predicate>>>
ReconstructPredicateWithPickedFields(
const std::shared_ptr<Predicate>& predicate,
const std::map<std::string, int32_t>& picked_field_name_to_idx);
diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h
index 28305c94..e62386c6 100644
--- a/include/paimon/read_context.h
+++ b/include/paimon/read_context.h
@@ -108,6 +108,7 @@ class PAIMON_EXPORT ReadContext {
return full_text_search_;
}
+ /// Whether a root TopK request is configured; full-text predicates are
not TopK requests.
bool HasVectorOrFullTextSearch() const {
return vector_search_ != nullptr || full_text_search_ != nullptr;
}
@@ -346,24 +347,37 @@ class PAIMON_EXPORT ReadContextBuilder {
/// It can significantly improve performance by reducing the amount of data
/// that needs to be read and processed.
///
+ /// All referenced fields, including full-text match fields, must be
projected in the read
+ /// schema.
+ /// TODO(lisizhuo.lsz): The behavior here will be aligned with Java Paimon
in a follow-up
+ /// change: fields referenced in predicates will no longer be required to
appear in the
+ /// projection. Full-text matches can be combined with scalar predicates
using
+ /// PredicateBuilder::And/Or. Full-text predicate execution is not
implemented yet; reading such
+ /// a predicate returns NotImplemented.
+ ///
/// @param predicate Shared pointer to the predicate for data filtering.
/// @return Reference to this builder for method chaining.
ReadContextBuilder& SetPredicate(const std::shared_ptr<Predicate>&
predicate);
- /// Set the vector search request used when reading data.
- ///
- /// TODO(jinli.zjw): Support combining vector search with predicates and
full-text search. Merge
- /// per-file candidates to compute read-level top-k results.
- ///
+ /// Configure the root vector TopK request, logically applied after the
predicate filter.
+ /// The search field need not be projected. SetVectorSearch is mutually
exclusive with
+ /// SetFullTextSearch and must have a positive limit.
+ /// Scores are required internally for TopK. Project
`_<field>_INDEX_SCORE` to include them
+ /// in the result. Per-field score output is not implemented yet; reading
such a projection
+ /// returns NotImplemented.
+ /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK
across files.
/// @param vector_search Vector search request, or nullptr to clear it.
/// @return Reference to this builder for method chaining.
ReadContextBuilder& SetVectorSearch(const std::shared_ptr<VectorSearch>&
vector_search);
- /// Set the full-text search request used when reading data.
- ///
- /// TODO(jinli.zjw): Support combining full-text search with predicates
and vector search. Apply
- /// the search limit to merged per-file results at read level.
- ///
+ /// Configure the root full-text TopK request, logically applied after the
predicate filter.
+ /// The search field need not be projected. Use
PredicateBuilder::FullTextMatch for a
+ /// full-text filter without TopK truncation. SetFullTextSearch is
mutually exclusive with
+ /// SetVectorSearch and must have a positive limit.
+ /// Scores are required internally for TopK. Project
`_<field>_INDEX_SCORE` to include them
+ /// in the result. Per-field score output is not implemented yet; reading
such a projection
+ /// returns NotImplemented.
+ /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK
across files.
/// @param full_text_search Full-text search request, or nullptr to clear
it.
/// @return Reference to this builder for method chaining.
ReadContextBuilder& SetFullTextSearch(const
std::shared_ptr<FullTextSearch>& full_text_search);
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index b3d1a1e2..33075614 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -118,6 +118,7 @@ set(PAIMON_COMMON_SRCS
common/predicate/contains.cpp
common/predicate/ends_with.cpp
common/predicate/equal.cpp
+ common/predicate/full_text_match_predicate.cpp
common/predicate/greater_or_equal.cpp
common/predicate/greater_than.cpp
common/predicate/in.cpp
@@ -673,6 +674,7 @@ if(PAIMON_BUILD_TESTS)
common/metrics/histogram_windowing_test.cpp
common/options/memory_size_test.cpp
common/options/time_duration_test.cpp
+ common/predicate/full_text_match_predicate_test.cpp
common/predicate/full_text_search_test.cpp
common/predicate/literal_converter_test.cpp
common/predicate/literal_test.cpp
diff --git a/src/paimon/common/file_index/file_index_reader.cpp
b/src/paimon/common/file_index/file_index_reader.cpp
index 9350e23f..7151c5e5 100644
--- a/src/paimon/common/file_index/file_index_reader.cpp
+++ b/src/paimon/common/file_index/file_index_reader.cpp
@@ -111,4 +111,9 @@ Result<std::shared_ptr<FileIndexResult>>
FileIndexReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
return Status::NotImplemented("This file index does not support full text
search");
}
+
+Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitFullTextMatch(
+ const std::shared_ptr<FullTextMatch>& full_text_match) {
+ return Status::NotImplemented("This file index does not support complete
full-text matching");
+}
} // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_reader_test.cpp
b/src/paimon/common/file_index/file_index_reader_test.cpp
index 86745c3e..40a25559 100644
--- a/src/paimon/common/file_index/file_index_reader_test.cpp
+++ b/src/paimon/common/file_index/file_index_reader_test.cpp
@@ -21,8 +21,11 @@
#include <utility>
#include "gtest/gtest.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/full_text_search.h"
#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/predicate/predicate_utils.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/testing/utils/testharness.h"
@@ -75,5 +78,40 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) {
std::make_shared<FullTextSearch>("body",
R"({"match":{"query":"word"}})", /*limit=*/1);
ASSERT_NOK_WITH_MSG(reader.VisitFullTextSearch(full_text_search),
"does not support full text search");
+ auto full_text_match = std::make_shared<FullTextMatch>("body",
R"({"match":{"query":"word"}})");
+ ASSERT_NOK_WITH_MSG(reader.VisitFullTextMatch(full_text_match),
+ "does not support complete full-text matching");
+}
+
+TEST(FileIndexReaderTest, TestFullTextMatchVisitorDispatch) {
+ class FullTextReader : public FileIndexReader {
+ public:
+ Result<std::shared_ptr<FileIndexResult>> VisitFullTextMatch(
+ const std::shared_ptr<FullTextMatch>& full_text_match) override {
+ received_match_ = full_text_match;
+ return FileIndexResult::Skip();
+ }
+
+ std::shared_ptr<FullTextMatch> received_match_;
+ };
+
+ auto match = std::make_shared<FullTextMatch>("body",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(0,
match));
+ auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate);
+ ASSERT_TRUE(leaf);
+
+ auto default_reader = std::make_shared<FileIndexReader>();
+ auto result =
+ PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(leaf,
default_reader);
+ ASSERT_NOK_WITH_MSG(result, "This file index does not support complete
full-text matching");
+ ASSERT_TRUE(result.status().IsNotImplemented());
+
+ auto reader = std::make_shared<FullTextReader>();
+ ASSERT_OK_AND_ASSIGN(
+ auto matches,
+ PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(leaf,
reader));
+ ASSERT_EQ(reader->received_match_, match);
+ ASSERT_OK_AND_ASSIGN(bool is_remain, matches->IsRemain());
+ ASSERT_FALSE(is_remain);
}
} // namespace paimon::test
diff --git a/src/paimon/common/predicate/full_text_match_function.h
b/src/paimon/common/predicate/full_text_match_function.h
new file mode 100644
index 00000000..6a36b4ac
--- /dev/null
+++ b/src/paimon/common/predicate/full_text_match_function.h
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "paimon/common/predicate/leaf_function.h"
+
+namespace paimon {
+
+/// Identifies an index-dependent predicate without implementing scalar
evaluation.
+class FullTextMatchFunction : public LeafFunction {
+ public:
+ static const FullTextMatchFunction& Instance() {
+ static const FullTextMatchFunction instance;
+ return instance;
+ }
+
+ Result<std::vector<char>> Test(const arrow::Array& array, const
std::vector<Literal>& literals,
+ arrow::MemoryPool* pool) const override {
+ return Status::NotImplemented(
+ "FullTextMatch does not support scalar Test; a full-text index is
required");
+ }
+
+ Result<bool> Test(const Literal& value, const std::vector<Literal>&
literals) const override {
+ return Status::NotImplemented(
+ "FullTextMatch does not support scalar Test; a full-text index is
required");
+ }
+
+ Result<bool> Test(int64_t row_count, const Literal& min_value, const
Literal& max_value,
+ const std::optional<int64_t>& null_count,
+ const std::vector<Literal>& literals) const override {
+ return Status::NotImplemented(
+ "FullTextMatch does not support statistics Test; a full-text index
is required");
+ }
+
+ const LeafFunction* Negate() const override {
+ return nullptr;
+ }
+
+ Type GetType() const override {
+ return Type::FULL_TEXT_MATCH;
+ }
+
+ std::string ToString() const override {
+ return "FullTextMatch";
+ }
+
+ private:
+ FullTextMatchFunction() = default;
+};
+
+} // namespace paimon
diff --git a/src/paimon/common/predicate/full_text_match_predicate.cpp
b/src/paimon/common/predicate/full_text_match_predicate.cpp
new file mode 100644
index 00000000..6f481c0c
--- /dev/null
+++ b/src/paimon/common/predicate/full_text_match_predicate.cpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/predicate/full_text_match_predicate.h"
+
+#include "fmt/format.h"
+#include "paimon/common/predicate/full_text_match_function.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/full_text_match.h"
+
+namespace paimon {
+
+FullTextMatchPredicate::FullTextMatchPredicate(int32_t field_index,
+ const
std::shared_ptr<FullTextMatch>& match)
+ : LeafPredicate(FullTextMatchFunction::Instance(), field_index,
match->field_name,
+ FieldType::STRING, {}),
+ full_text_match_(match) {}
+
+bool FullTextMatchPredicate::operator==(const Predicate& other) const {
+ if (this == &other) {
+ return true;
+ }
+ auto text_predicate =
dynamic_cast<FullTextMatchPredicate*>(const_cast<Predicate*>(&other));
+ if (!text_predicate) {
+ return false;
+ }
+ return LeafPredicate::operator==(other) &&
+ *full_text_match_ == *text_predicate->full_text_match_;
+}
+
+std::string FullTextMatchPredicate::ToString() const {
+ return fmt::format(
+ "FullTextMatch({}, {}, include_row_ids={})", FieldName(),
full_text_match_->query,
+ full_text_match_->include_row_ids ?
full_text_match_->include_row_ids->ToString() : "null");
+}
+
+std::shared_ptr<FullTextMatchPredicate>
FullTextMatchPredicate::NewLeafPredicate(
+ int32_t new_field_index) const {
+ return std::make_shared<FullTextMatchPredicate>(new_field_index,
full_text_match_);
+}
+
+std::shared_ptr<FullTextMatchPredicate>
FullTextMatchPredicate::NewLeafPredicate(
+ const std::string& new_field_name) const {
+ auto request = std::make_shared<FullTextMatch>(*full_text_match_);
+ request->field_name = new_field_name;
+ return std::make_shared<FullTextMatchPredicate>(FieldIndex(), request);
+}
+
+} // namespace paimon
diff --git a/src/paimon/common/predicate/full_text_match_predicate.h
b/src/paimon/common/predicate/full_text_match_predicate.h
new file mode 100644
index 00000000..61ae13a7
--- /dev/null
+++ b/src/paimon/common/predicate/full_text_match_predicate.h
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "paimon/predicate/leaf_predicate.h"
+
+namespace paimon {
+struct FullTextMatch;
+
+/// A full-text leaf evaluated by an index, rather than scalar
PredicateFilter::Test.
+/// Use PredicateBuilder::FullTextMatch to construct it.
+class FullTextMatchPredicate : public LeafPredicate {
+ public:
+ FullTextMatchPredicate(int32_t field_index, const
std::shared_ptr<FullTextMatch>& match);
+
+ const std::shared_ptr<FullTextMatch>& GetFullTextMatch() const {
+ return full_text_match_;
+ }
+
+ bool operator==(const Predicate& other) const override;
+ std::string ToString() const override;
+
+ /// Rebind the field position while sharing the original request.
+ std::shared_ptr<FullTextMatchPredicate> NewLeafPredicate(int32_t
new_field_index) const;
+ /// Rebind the field name using a request copy, preserving the original
request.
+ std::shared_ptr<FullTextMatchPredicate> NewLeafPredicate(
+ const std::string& new_field_name) const;
+
+ private:
+ std::shared_ptr<FullTextMatch> full_text_match_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/common/predicate/full_text_match_predicate_test.cpp
b/src/paimon/common/predicate/full_text_match_predicate_test.cpp
new file mode 100644
index 00000000..cc6ddfc8
--- /dev/null
+++ b/src/paimon/common/predicate/full_text_match_predicate_test.cpp
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/predicate/full_text_match_predicate.h"
+
+#include "arrow/api.h"
+#include "gtest/gtest.h"
+#include "paimon/common/predicate/full_text_match_function.h"
+#include "paimon/common/predicate/predicate_filter.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/full_text_match.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<FullTextMatch> MakeMatch(
+ const std::string& field_name = "title",
+ const std::string& query = R"({"match":{"query":"apple"}})") {
+ return std::make_shared<FullTextMatch>(field_name, query);
+}
+
+} // namespace
+
+TEST(FullTextMatchPredicateTest, TestBasicAndComposesWithScalarPredicates) {
+ auto match = MakeMatch();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::FullTextMatch(1, match));
+ auto text = std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate);
+ ASSERT_TRUE(text);
+ ASSERT_EQ(text->GetFullTextMatch(), match);
+ ASSERT_EQ(text->FieldIndex(), 1);
+ ASSERT_EQ(text->FieldName(), "title");
+ ASSERT_EQ(text->GetFieldType(), FieldType::STRING);
+ ASSERT_EQ(text->GetFunction().GetType(), Function::Type::FULL_TEXT_MATCH);
+ ASSERT_TRUE(text->Literals().empty());
+ ASSERT_FALSE(std::dynamic_pointer_cast<PredicateFilter>(predicate));
+ ASSERT_NE(text->ToString().find(match->query), std::string::npos);
+
+ auto scalar = PredicateBuilder::LessThan(0, "price", FieldType::INT,
Literal(10));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> conjunction,
+ PredicateBuilder::And({scalar, predicate}));
+ auto compound = std::dynamic_pointer_cast<CompoundPredicate>(conjunction);
+ ASSERT_TRUE(compound);
+ ASSERT_EQ(compound->Children(),
(std::vector<std::shared_ptr<Predicate>>{scalar, predicate}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> disjunction,
+ PredicateBuilder::Or({scalar, predicate}));
+ ASSERT_EQ(disjunction->GetFunction().GetType(), Function::Type::OR);
+ ASSERT_FALSE(text->Negate());
+ ASSERT_NOK_WITH_MSG(PredicateBuilder::Not(predicate), "Could not construct
A NOT predicate");
+ ASSERT_NOK_WITH_MSG(PredicateBuilder::Not(conjunction), "Could not
construct A NOT predicate");
+}
+
+TEST(FullTextMatchPredicateTest, TestRebind) {
+ auto match = std::make_shared<FullTextMatch>("title",
R"({"match":{"query":"apple"}})",
+ RoaringBitmap64::From({1, 3,
9}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::FullTextMatch(1, match));
+ auto text = std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate);
+ ASSERT_TRUE(text);
+
+ auto picked = text->NewLeafPredicate(0);
+ ASSERT_EQ(picked->FieldIndex(), 0);
+ ASSERT_EQ(picked->GetFullTextMatch(), match);
+ ASSERT_EQ(text->FieldIndex(), 1);
+
+ auto renamed = text->NewLeafPredicate("stored_title");
+ ASSERT_EQ(renamed->FieldIndex(), 1);
+ ASSERT_EQ(renamed->FieldName(), "stored_title");
+ ASSERT_EQ(renamed->GetFullTextMatch()->field_name, "stored_title");
+ ASSERT_EQ(renamed->GetFullTextMatch()->query, match->query);
+ ASSERT_EQ(renamed->GetFullTextMatch()->include_row_ids,
match->include_row_ids);
+ ASSERT_NE(renamed->GetFullTextMatch(), match);
+ ASSERT_EQ(match->field_name, "title");
+ ASSERT_EQ(text->FieldName(), "title");
+}
+
+TEST(FullTextMatchPredicateTest, TestEquality) {
+ auto match = MakeMatch();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::FullTextMatch(1, match));
+ auto text = std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate);
+ ASSERT_TRUE(text);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> same,
+ PredicateBuilder::FullTextMatch(1, MakeMatch()));
+ ASSERT_TRUE(*predicate == *same);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<Predicate> other_query,
+ PredicateBuilder::FullTextMatch(1, MakeMatch("title",
R"({"match":{"query":"orange"}})")));
+ ASSERT_FALSE(*predicate == *other_query);
+ auto restricted = std::make_shared<FullTextMatch>("title",
R"({"match":{"query":"apple"}})",
+
RoaringBitmap64::From({3, 9}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> candidates,
+ PredicateBuilder::FullTextMatch(1, restricted));
+ ASSERT_FALSE(*predicate == *candidates);
+
ASSERT_NE(candidates->ToString().find(restricted->include_row_ids->ToString()),
+ std::string::npos);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> other_field,
+ PredicateBuilder::FullTextMatch(1,
MakeMatch("body")));
+ ASSERT_FALSE(*predicate == *other_field);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> other_index,
+ PredicateBuilder::FullTextMatch(0, MakeMatch()));
+ ASSERT_FALSE(*predicate == *other_index);
+}
+
+TEST(FullTextMatchPredicateTest, TestRejectsInvalidRequests) {
+ ASSERT_NOK_WITH_MSG(PredicateBuilder::FullTextMatch(0, nullptr),
+ "FullTextMatch request must not be null");
+ ASSERT_NOK_WITH_MSG(PredicateBuilder::FullTextMatch(0, MakeMatch("")),
+ "FullTextMatch field name must not be empty");
+ ASSERT_NOK_WITH_MSG(PredicateBuilder::FullTextMatch(0, MakeMatch("title",
"")),
+ "FullTextMatch query must not be empty");
+}
+
+TEST(FullTextMatchPredicateTest, TestFunctionCannotEvaluateRowsOrStatistics) {
+ const auto& function = FullTextMatchFunction::Instance();
+ arrow::NullArray array(2);
+ ASSERT_NOK_WITH_MSG(
+ function.Test(array, {}, arrow::default_memory_pool()),
+ "FullTextMatch does not support scalar Test; a full-text index is
required");
+ ASSERT_NOK_WITH_MSG(
+ function.Test(Literal(1), {}),
+ "FullTextMatch does not support scalar Test; a full-text index is
required");
+ ASSERT_NOK_WITH_MSG(
+ function.Test(2, Literal(1), Literal(2), /*null_count=*/0, {}),
+ "FullTextMatch does not support statistics Test; a full-text index is
required");
+ ASSERT_FALSE(function.Negate());
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/common/predicate/predicate_builder.cpp
b/src/paimon/common/predicate/predicate_builder.cpp
index 529905e6..545fd7f4 100644
--- a/src/paimon/common/predicate/predicate_builder.cpp
+++ b/src/paimon/common/predicate/predicate_builder.cpp
@@ -25,6 +25,7 @@
#include "paimon/common/predicate/contains.h"
#include "paimon/common/predicate/ends_with.h"
#include "paimon/common/predicate/equal.h"
+#include "paimon/common/predicate/full_text_match_predicate.h"
#include "paimon/common/predicate/greater_or_equal.h"
#include "paimon/common/predicate/greater_than.h"
#include "paimon/common/predicate/in.h"
@@ -38,12 +39,27 @@
#include "paimon/common/predicate/not_in.h"
#include "paimon/common/predicate/or.h"
#include "paimon/common/predicate/starts_with.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/literal.h"
#include "paimon/status.h"
namespace paimon {
enum class FieldType;
+Result<std::shared_ptr<Predicate>> PredicateBuilder::FullTextMatch(
+ int32_t field_index, const std::shared_ptr<paimon::FullTextMatch>& match) {
+ if (!match) {
+ return Status::Invalid("FullTextMatch request must not be null");
+ }
+ if (match->field_name.empty()) {
+ return Status::Invalid("FullTextMatch field name must not be empty");
+ }
+ if (match->query.empty()) {
+ return Status::Invalid("FullTextMatch query must not be empty");
+ }
+ return std::make_shared<FullTextMatchPredicate>(field_index, match);
+}
+
// TODO(xinyu.lxy): predicate field_index use index in read schema now, but
java paimon use index
// in file schema
std::shared_ptr<Predicate> PredicateBuilder::Equal(int32_t field_index,
diff --git a/src/paimon/common/predicate/predicate_utils.cpp
b/src/paimon/common/predicate/predicate_utils.cpp
index ab9c8f4a..f8dc2500 100644
--- a/src/paimon/common/predicate/predicate_utils.cpp
+++ b/src/paimon/common/predicate/predicate_utils.cpp
@@ -21,12 +21,37 @@
#include <utility>
#include "paimon/common/predicate/compound_predicate_impl.h"
+#include "paimon/common/predicate/full_text_match_predicate.h"
#include "paimon/common/predicate/leaf_predicate_impl.h"
#include "paimon/predicate/compound_predicate.h"
#include "paimon/predicate/leaf_predicate.h"
#include "paimon/predicate/predicate_builder.h"
namespace paimon {
+
+Result<std::shared_ptr<FullTextMatch>> PredicateUtils::GetFullTextMatch(
+ const std::shared_ptr<LeafPredicate>& predicate) {
+ auto full_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate);
+ if (!full_text) {
+ return Status::Invalid("FULL_TEXT_MATCH requires a
FullTextMatchPredicate");
+ }
+ return full_text->GetFullTextMatch();
+}
+
+bool PredicateUtils::ContainsFullTextMatch(const std::shared_ptr<Predicate>&
predicate) {
+ if (std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate)) {
+ return true;
+ }
+ if (auto compound =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
+ for (const auto& child : compound->Children()) {
+ if (ContainsFullTextMatch(child)) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
Result<bool> PredicateUtils::ContainAnyField(const std::shared_ptr<Predicate>&
predicate,
const std::set<std::string>&
field_names) {
if (auto leaf_predicate =
std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
@@ -135,6 +160,13 @@ PredicateUtils::ReconstructPredicateWithPickedFields(
}
return std::optional<std::shared_ptr<Predicate>>(
compound_predicate->NewCompoundPredicate(mapped_children));
+ } else if (auto full_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate)) {
+ auto iter = picked_field_name_to_idx.find(full_text->FieldName());
+ if (iter == picked_field_name_to_idx.end()) {
+ return std::optional<std::shared_ptr<Predicate>>();
+ }
+ return std::optional<std::shared_ptr<Predicate>>(
+ full_text->NewLeafPredicate(/*new_field_index=*/iter->second));
} else if (auto leaf_predicate =
std::dynamic_pointer_cast<LeafPredicateImpl>(predicate)) {
auto iter = picked_field_name_to_idx.find(leaf_predicate->FieldName());
if (iter == picked_field_name_to_idx.end()) {
diff --git a/src/paimon/common/predicate/predicate_utils_test.cpp
b/src/paimon/common/predicate/predicate_utils_test.cpp
index 3d22628a..8b6a98a8 100644
--- a/src/paimon/common/predicate/predicate_utils_test.cpp
+++ b/src/paimon/common/predicate/predicate_utils_test.cpp
@@ -21,12 +21,55 @@
#include <utility>
#include "gtest/gtest.h"
+#include "paimon/common/predicate/full_text_match_predicate.h"
#include "paimon/defs.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
+TEST(PredicateUtilsTest, TestFullTextMatchFieldUtilities) {
+ auto match = std::make_shared<FullTextMatch>("text",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> text,
+ PredicateBuilder::FullTextMatch(1, match));
+ auto scalar = PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(7));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::And({scalar, text}));
+ ASSERT_TRUE(PredicateUtils::ContainsFullTextMatch(predicate));
+ ASSERT_FALSE(PredicateUtils::ContainsFullTextMatch(scalar));
+ ASSERT_FALSE(PredicateUtils::ContainsFullTextMatch(nullptr));
+ std::set<std::string> names;
+ ASSERT_OK(PredicateUtils::GetAllNames(predicate, &names));
+ ASSERT_EQ(names, (std::set<std::string>{"id", "text"}));
+ ASSERT_OK_AND_ASSIGN(bool contains,
PredicateUtils::ContainAnyField(predicate, {"text"}));
+ ASSERT_TRUE(contains);
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> picked,
+ PredicateUtils::CreatePickedFieldFilter(predicate,
{{"text", 4}}));
+ auto picked_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(picked);
+ ASSERT_TRUE(picked_text);
+ ASSERT_EQ(picked_text->FieldIndex(), 4);
+ ASSERT_EQ(picked_text->GetFullTextMatch(), match);
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> disjunction,
+ PredicateBuilder::Or({scalar, text}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> missing,
+ PredicateUtils::CreatePickedFieldFilter(disjunction,
{{"text", 4}}));
+ ASSERT_FALSE(missing);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<Predicate> remapped,
+ PredicateUtils::CreatePickedFieldFilter(disjunction, {{"id", 2},
{"text", 4}}));
+ auto compound = std::dynamic_pointer_cast<CompoundPredicate>(remapped);
+ ASSERT_TRUE(compound);
+ ASSERT_EQ(compound->GetFunction().GetType(), Function::Type::OR);
+ picked_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(compound->Children()[1]);
+ ASSERT_TRUE(picked_text);
+ ASSERT_EQ(picked_text->FieldIndex(), 4);
+ ASSERT_EQ(picked_text->GetFullTextMatch(), match);
+}
+
TEST(PredicateUtilsTest, TestSplitAnd) {
ASSERT_OK_AND_ASSIGN(
auto child1,
diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp
b/src/paimon/common/predicate/predicate_validator_test.cpp
index 75ae141c..89432b72 100644
--- a/src/paimon/common/predicate/predicate_validator_test.cpp
+++ b/src/paimon/common/predicate/predicate_validator_test.cpp
@@ -22,6 +22,7 @@
#include "gtest/gtest.h"
#include "paimon/data/decimal.h"
#include "paimon/data/timestamp.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/result.h"
#include "paimon/testing/utils/testharness.h"
@@ -31,6 +32,48 @@ class Schema;
} // namespace arrow
namespace paimon::test {
+TEST(PredicateValidatorTest, TestFullTextMatchRequiresProjectedStringField) {
+ auto schema =
+ arrow::schema({arrow::field("id", arrow::int32()),
arrow::field("text", arrow::utf8())});
+ auto match = std::make_shared<FullTextMatch>("text",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> text,
+ PredicateBuilder::FullTextMatch(1, match));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, text,
true));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> wrong_index,
+ PredicateBuilder::FullTextMatch(0, match));
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, wrong_index,
true),
+ "field text has field idx 1 in input schema, mismatch field idx 0 in
predicate");
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
wrong_index, false));
+ auto missing = arrow::schema({arrow::field("id", arrow::int32())});
+
ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema(*missing,
text, true),
+ "field text does not exist in schema");
+ auto numeric =
+ arrow::schema({arrow::field("id", arrow::int32()),
arrow::field("text", arrow::int32())});
+
ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema(*numeric,
text, true),
+ "schema type int32 mismatches predicate field type
STRING");
+}
+
+TEST(PredicateValidatorTest, TestFullTextMatchAllowsOrAndRepeatedFields) {
+ auto schema =
+ arrow::schema({arrow::field("id", arrow::int32()),
arrow::field("text", arrow::utf8())});
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<Predicate> apple,
+ PredicateBuilder::FullTextMatch(
+ 1, std::make_shared<FullTextMatch>("text",
R"({"match":{"query":"apple"}})")));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<Predicate> orange,
+ PredicateBuilder::FullTextMatch(
+ 1, std::make_shared<FullTextMatch>("text",
R"({"match":{"query":"orange"}})")));
+ auto scalar = PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(7));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> conjunction,
+ PredicateBuilder::And({apple, orange}));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
conjunction, true));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> disjunction,
+ PredicateBuilder::Or({conjunction, scalar}));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
disjunction, true));
+}
+
TEST(PredicateValidatorTest, TestValidateLiterals) {
std::string str("apple");
std::shared_ptr<arrow::Schema> schema = arrow::schema(arrow::FieldVector({
diff --git a/src/paimon/common/table/special_fields.h
b/src/paimon/common/table/special_fields.h
index f3e64fcc..73562140 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -20,6 +20,7 @@
#include <cstdint>
#include <limits>
+#include <optional>
#include <string>
#include <utility>
@@ -67,6 +68,20 @@ struct SpecialFields {
return data_field;
}
+ static std::string IndexScoreFieldName(const std::string& field_name) {
+ return "_" + field_name + "_INDEX_SCORE";
+ }
+
+ /// Parse a per-field score projection. The global `_INDEX_SCORE` is
separate.
+ static std::optional<std::string> ParseIndexScoreFieldName(const
std::string& name) {
+ const std::string suffix = "_INDEX_SCORE";
+ if (name.size() <= suffix.size() + 1 || !StringUtils::StartsWith(name,
"_") ||
+ !StringUtils::EndsWith(name, suffix)) {
+ return std::nullopt;
+ }
+ return name.substr(1, name.size() - suffix.size() - 1);
+ }
+
static const DataField& RealtimeOffset() {
static const DataField data_field =
DataField(SpecialFieldIds::REALTIME_OFFSET,
@@ -93,7 +108,8 @@ struct SpecialFields {
}
return field_name == SequenceNumber().Name() || field_name ==
ValueKind().Name() ||
field_name == RowKind().Name() || field_name == RowId().Name()
||
- field_name == IndexScore().Name();
+ field_name == IndexScore().Name() ||
+ ParseIndexScoreFieldName(field_name).has_value();
}
// TODO(xinyu.lxy): add a func to complete row-tracking fields
diff --git a/src/paimon/common/table/special_fields_test.cpp
b/src/paimon/common/table/special_fields_test.cpp
index 05c0808e..94df2167 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -55,6 +55,19 @@ TEST(SpecialFieldsTest, TestIndexScore) {
ASSERT_EQ(SpecialFields::IndexScore().Type()->id(), arrow::Type::FLOAT);
}
+TEST(SpecialFieldsTest, TestPerFieldIndexScoreNames) {
+ for (const auto& field_name : {"F1", "body_text", "_title", "文档"}) {
+ const auto name = SpecialFields::IndexScoreFieldName(field_name);
+ ASSERT_EQ(SpecialFields::ParseIndexScoreFieldName(name), field_name);
+ ASSERT_TRUE(SpecialFields::IsSystemField(name));
+ }
+ ASSERT_EQ(SpecialFields::IndexScoreFieldName("F1"), "_F1_INDEX_SCORE");
+ for (const auto& name : {"_INDEX_SCORE", "__INDEX_SCORE",
"F1_INDEX_SCORE", "_F1_index_score",
+ "_F1_INDEX_SCORE_extra"}) {
+ ASSERT_FALSE(SpecialFields::ParseIndexScoreFieldName(name));
+ }
+}
+
TEST(SpecialFieldsTest, TestRealtimeOffset) {
ASSERT_EQ(SpecialFields::RealtimeOffset().Id(),
SpecialFieldIds::REALTIME_OFFSET);
ASSERT_EQ(SpecialFields::RealtimeOffset().Name(), "_REALTIME_OFFSET");
diff --git a/src/paimon/core/operation/file_store_scan.cpp
b/src/paimon/core/operation/file_store_scan.cpp
index e5f13233..4e508bab 100644
--- a/src/paimon/core/operation/file_store_scan.cpp
+++ b/src/paimon/core/operation/file_store_scan.cpp
@@ -690,6 +690,9 @@ Result<bool>
FileStoreScan::FilterNonPartitionManifestEntry(const ManifestEntry&
Status FileStoreScan::SplitAndSetFilter(const std::vector<std::string>&
partition_keys,
const std::shared_ptr<arrow::Schema>&
arrow_schema,
const std::shared_ptr<ScanFilter>&
scan_filters) {
+ if (PredicateUtils::ContainsFullTextMatch(scan_filters->GetPredicate())) {
+ return Status::NotImplemented("FullTextMatch predicate execution is
not implemented yet");
+ }
if (scan_filters->GetPredicate()) {
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FieldMappingBuilder>
mapping_builder,
FieldMappingBuilder::Create(arrow_schema,
partition_keys,
diff --git a/src/paimon/core/operation/file_store_scan_test.cpp
b/src/paimon/core/operation/file_store_scan_test.cpp
index ef784f76..5c53f138 100644
--- a/src/paimon/core/operation/file_store_scan_test.cpp
+++ b/src/paimon/core/operation/file_store_scan_test.cpp
@@ -29,10 +29,12 @@
#include "paimon/core/manifest/file_source.h"
#include "paimon/core/manifest/manifest_entry_serializer.h"
#include "paimon/core/manifest/snapshot_live_manifest_entries.h"
+#include "paimon/core/operation/append_only_file_store_scan.h"
#include "paimon/core/stats/simple_stats.h"
#include "paimon/data/timestamp.h"
#include "paimon/memory/bytes.h"
#include "paimon/memory/memory_segment.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/testing/utils/binary_row_generator.h"
#include "paimon/testing/utils/testharness.h"
@@ -140,6 +142,28 @@ TEST_F(FileStoreScanTest,
TestCreatePartitionPredicateWithInvalidPartitionFilter
"field invalid does not exist in partition keys");
}
+TEST_F(FileStoreScanTest,
TestFullTextMatchDoesNotEnterPartitionOrStatsFilters) {
+ auto match = std::make_shared<FullTextMatch>("f0",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(auto full_text,
PredicateBuilder::FullTextMatch(/*field_index=*/0, match));
+ auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1",
FieldType::INT, Literal(1));
+ ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar,
full_text}));
+ auto filters = std::make_shared<ScanFilter>(
+ predicate, std::vector<std::map<std::string, std::string>>{},
std::nullopt);
+ ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
+ for (const std::vector<std::string>& partition_keys :
+ {std::vector<std::string>{}, std::vector<std::string>{"f0"}}) {
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<TableSchema> table_schema,
+ TableSchema::Create(/*schema_id=*/0, schema_, partition_keys,
/*primary_keys=*/{}, {}));
+ auto result = AppendOnlyFileStoreScan::Create(
+ /*snapshot_manager=*/nullptr, /*schema_manager=*/nullptr,
/*manifest_list=*/nullptr,
+ /*manifest_file=*/nullptr, table_schema, schema_, filters, options,
+ CreateDefaultExecutor(), GetDefaultPool());
+ ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not
implemented yet");
+ ASSERT_TRUE(result.status().IsNotImplemented());
+ }
+}
+
TEST_F(FileStoreScanTest, TestFilterManifestByRowRanges) {
class FakeFileStoreScan : public FileStoreScan {
public:
diff --git a/src/paimon/core/operation/internal_read_context.cpp
b/src/paimon/core/operation/internal_read_context.cpp
index 429302cc..a89eb0ae 100644
--- a/src/paimon/core/operation/internal_read_context.cpp
+++ b/src/paimon/core/operation/internal_read_context.cpp
@@ -18,6 +18,8 @@
#include "paimon/core/operation/internal_read_context.h"
+#include <algorithm>
+#include <functional>
#include <optional>
#include <utility>
@@ -27,6 +29,7 @@
#include "fmt/format.h"
#include "paimon/common/data/variant/variant_access_utils.h"
#include "paimon/common/data/variant/variant_type_utils.h"
+#include "paimon/common/predicate/full_text_match_predicate.h"
#include "paimon/common/predicate/predicate_validator.h"
#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
@@ -36,7 +39,9 @@
#include "paimon/core/options/map_storage_layout.h"
#include "paimon/core/schema/arrow_schema_validator.h"
#include "paimon/core/utils/nested_projection_utils.h"
+#include "paimon/predicate/compound_predicate.h"
#include "paimon/predicate/full_text_search.h"
+#include "paimon/predicate/predicate_utils.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/status.h"
@@ -175,7 +180,12 @@ std::optional<DataField>
InternalReadContext::TryResolveSpecialFieldById(
}
std::optional<DataField> InternalReadContext::TryResolveSpecialFieldByName(
- const std::string& name, const CoreOptions& core_options, bool
has_vector_or_full_text_search) {
+ const std::string& name, const CoreOptions& core_options, bool
has_vector_or_full_text_search,
+ int32_t output_field_idx) {
+ if (SpecialFields::ParseIndexScoreFieldName(name)) {
+ // These virtual columns have read-local negative IDs, not persistent
table IDs.
+ return DataField(-1 - output_field_idx, arrow::field(name,
arrow::float32()));
+ }
if (name == SpecialFields::ValueKind().Name()) {
return SpecialFields::ValueKind();
}
@@ -245,7 +255,8 @@ Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::Create(
// Align special-field validation with read_field_ids/read_field_names
branches.
for (const auto& read_field : read_schema->fields()) {
if (auto resolved_special_field = TryResolveSpecialFieldByName(
- read_field->name(), core_options,
has_vector_or_full_text_search)) {
+ read_field->name(), core_options,
has_vector_or_full_text_search,
+ static_cast<int32_t>(read_data_fields.size()))) {
read_data_fields.push_back(*resolved_special_field);
continue;
}
@@ -281,8 +292,9 @@ Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::Create(
} else if (!context->GetReadFieldNames().empty()) {
read_data_fields.reserve(context->GetReadFieldNames().size());
for (const auto& name : context->GetReadFieldNames()) {
- if (auto resolved_special_field = TryResolveSpecialFieldByName(
- name, core_options, has_vector_or_full_text_search)) {
+ if (auto resolved_special_field =
+ TryResolveSpecialFieldByName(name, core_options,
has_vector_or_full_text_search,
+
static_cast<int32_t>(read_data_fields.size()))) {
read_data_fields.push_back(*resolved_special_field);
continue;
}
@@ -311,13 +323,85 @@ Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::Create(
PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
*read_schema, context->GetPredicate(),
/*validate_field_idx=*/true));
}
-
if (!context->GetMemoryPool()) {
return Status::Invalid("memory pool is null");
}
std::shared_ptr<arrow::MemoryPool> arrow_pool =
GetArrowPool(context->GetMemoryPool());
- return std::unique_ptr<InternalReadContext>(
+ auto internal_context = std::unique_ptr<InternalReadContext>(
new InternalReadContext(context, table_schema, read_schema,
core_options, arrow_pool));
+ PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores());
+ return internal_context;
+}
+
+Status InternalReadContext::BindIndexScores() {
+ std::map<std::string, int32_t> projected_scores;
+ for (int32_t idx = 0; idx < read_schema_->num_fields(); ++idx) {
+ if (auto field_name =
+
SpecialFields::ParseIndexScoreFieldName(read_schema_->field(idx)->name())) {
+ projected_scores.emplace(*field_name, idx);
+ }
+ }
+ if (projected_scores.empty()) {
+ return Status::OK();
+ }
+ auto bind = [&](const IndexScoreSource& source, bool under_or) -> Status {
+ auto iter = projected_scores.find(source.field_name);
+ if (iter == projected_scores.end()) {
+ return Status::OK();
+ }
+ if (under_or) {
+ return Status::Invalid("Score-producing FullTextMatch cannot
appear under OR");
+ }
+ if (!index_score_bindings_.emplace(iter->second, source).second) {
+ return Status::Invalid(fmt::format(
+ "Score-producing search field '{}' occurs more than once",
source.field_name));
+ }
+ return Status::OK();
+ };
+ std::function<Status(const std::shared_ptr<Predicate>&, bool)> visit =
+ [&](const std::shared_ptr<Predicate>& predicate, bool under_or) ->
Status {
+ if (auto full_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate)) {
+ return bind({IndexScoreSource::Type::FULL_TEXT_MATCH,
full_text->FieldName()},
+ under_or);
+ }
+ if (auto compound =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
+ const bool child_under_or =
+ under_or || compound->GetFunction().GetType() ==
Function::Type::OR;
+ for (const auto& child : compound->Children()) {
+ PAIMON_RETURN_NOT_OK(visit(child, child_under_or));
+ }
+ }
+ return Status::OK();
+ };
+ PAIMON_RETURN_NOT_OK(visit(GetPredicate(), /*under_or=*/false));
+ if (const auto& search = GetVectorSearch()) {
+ PAIMON_RETURN_NOT_OK(
+ bind({IndexScoreSource::Type::VECTOR_SEARCH, search->field_name},
/*under_or=*/false));
+ }
+ if (const auto& search = GetFullTextSearch()) {
+ PAIMON_RETURN_NOT_OK(bind({IndexScoreSource::Type::FULL_TEXT_SEARCH,
search->field_name},
+ /*under_or=*/false));
+ }
+ for (const auto& [field_name, output_idx] : projected_scores) {
+ if (index_score_bindings_.find(output_idx) ==
index_score_bindings_.end()) {
+ return Status::Invalid(fmt::format("Index score field '{}' has no
search source",
+
read_schema_->field(output_idx)->name()));
+ }
+ }
+ return Status::OK();
+}
+
+bool InternalReadContext::NeedsScores(const IndexScoreSource& source) const {
+ if (source.type == IndexScoreSource::Type::VECTOR_SEARCH) {
+ const auto& search = GetVectorSearch();
+ return search && search->field_name == source.field_name;
+ }
+ if (source.type == IndexScoreSource::Type::FULL_TEXT_SEARCH) {
+ const auto& search = GetFullTextSearch();
+ return search && search->field_name == source.field_name;
+ }
+ return std::any_of(index_score_bindings_.begin(),
index_score_bindings_.end(),
+ [&source](const auto& binding) { return binding.second
== source; });
}
InternalReadContext::InternalReadContext(const std::shared_ptr<ReadContext>&
read_context,
@@ -334,6 +418,11 @@ InternalReadContext::InternalReadContext(const
std::shared_ptr<ReadContext>& rea
Result<std::shared_ptr<InternalReadContext>>
InternalReadContext::CreateWithSchema(
const std::shared_ptr<InternalReadContext>& original,
const std::shared_ptr<arrow::Schema>& new_read_schema) {
+ if (original->HasVectorOrFullTextSearch() ||
+ PredicateUtils::ContainsFullTextMatch(original->GetPredicate())) {
+ return Status::NotImplemented(
+ "CreateWithSchema does not support search requests or index score
projections");
+ }
// Create a new InternalReadContext sharing all properties except
read_schema.
// The new read_schema is the minimal column set for COUNT(*).
return std::shared_ptr<InternalReadContext>(
diff --git a/src/paimon/core/operation/internal_read_context.h
b/src/paimon/core/operation/internal_read_context.h
index 8e34a85b..d3601fdf 100644
--- a/src/paimon/core/operation/internal_read_context.h
+++ b/src/paimon/core/operation/internal_read_context.h
@@ -43,11 +43,31 @@ class RealtimeContext;
// internal read context, contains ReadContext, CoreOptions and TableSchema
class InternalReadContext {
public:
+ struct IndexScoreSource {
+ enum class Type { VECTOR_SEARCH, FULL_TEXT_SEARCH, FULL_TEXT_MATCH };
+
+ bool operator==(const IndexScoreSource& other) const {
+ return type == other.type && field_name == other.field_name;
+ }
+
+ Type type;
+ /// Original table field name, before per-file field mapping.
+ std::string field_name;
+ };
+
static Result<std::unique_ptr<InternalReadContext>> Create(
const std::shared_ptr<ReadContext>& read_context,
const std::shared_ptr<TableSchema>& table_schema,
const std::map<std::string, std::string>& options);
+ /// Maps score output positions in the read schema to their logical search
sources.
+ const std::map<int32_t, IndexScoreSource>& GetIndexScoreBindings() const {
+ return index_score_bindings_;
+ }
+
+ /// Root TopK needs scores even without a score projection.
+ bool NeedsScores(const IndexScoreSource& source) const;
+
const CoreOptions& GetCoreOptions() const {
return options_;
}
@@ -129,10 +149,8 @@ class InternalReadContext {
return read_context_->GetWarmupLevel();
}
- /// Create a new InternalReadContext with a different read schema.
- /// Useful for creating a context with a minimal column set for
specialized reads.
- /// All other settings (predicate, options, table_schema, etc.) are
inherited
- /// from the original context.
+ /// Create the minimal read schema for PK COUNT(*). Search requests and
score
+ /// projections are unsupported; their bindings cannot be inherited across
schemas.
static Result<std::shared_ptr<InternalReadContext>> CreateWithSchema(
const std::shared_ptr<InternalReadContext>& original,
const std::shared_ptr<arrow::Schema>& new_read_schema);
@@ -149,16 +167,19 @@ class InternalReadContext {
bool
has_vector_or_full_text_search);
static std::optional<DataField> TryResolveSpecialFieldByName(
const std::string& name, const CoreOptions& core_options,
- bool has_vector_or_full_text_search);
+ bool has_vector_or_full_text_search, int32_t output_field_idx);
static Result<std::shared_ptr<arrow::Field>>
AlignReadFieldWithTableFieldIds(
const std::shared_ptr<arrow::Field>& read_field,
const std::shared_ptr<arrow::Field>& table_field);
+ Status BindIndexScores();
+
std::shared_ptr<ReadContext> read_context_;
std::shared_ptr<TableSchema> table_schema_;
std::shared_ptr<arrow::Schema> read_schema_;
std::shared_ptr<arrow::MemoryPool> arrow_pool_;
CoreOptions options_;
+ std::map<int32_t, IndexScoreSource> index_score_bindings_;
};
} // namespace paimon
diff --git a/src/paimon/core/operation/internal_read_context_test.cpp
b/src/paimon/core/operation/internal_read_context_test.cpp
index 086d56d1..ca7b79dc 100644
--- a/src/paimon/core/operation/internal_read_context_test.cpp
+++ b/src/paimon/core/operation/internal_read_context_test.cpp
@@ -27,6 +27,7 @@
#include "arrow/c/bridge.h"
#include "arrow/type.h"
+#include "fmt/format.h"
#include "gtest/gtest.h"
#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
@@ -34,7 +35,9 @@
#include "paimon/data/shredding/map_shared_shredding_schema_utils.h"
#include "paimon/defs.h"
#include "paimon/fs/local/local_file_system.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/full_text_search.h"
+#include "paimon/predicate/predicate_builder.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/status.h"
#include "paimon/testing/utils/testharness.h"
@@ -278,6 +281,332 @@ TEST(InternalReadContext,
TestFileFullTextSearchScoreIsNotSupported) {
"does not support score output");
}
+TEST(InternalReadContext, TestFullTextMatchValidatesProjection) {
+ std::string path = paimon::test::GetDataDir() +
"/orc/append_09.db/append_09";
+ SchemaManager schema_manager(std::make_shared<LocalFileSystem>(), path);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
schema_manager.ReadSchema(0));
+ struct TestCase {
+ int32_t field_index;
+ std::string field_name;
+ std::vector<std::string> projection;
+ std::string expected_error;
+ };
+ const std::vector<TestCase> cases = {
+ {0, "f0", {"f0"}, ""},
+ {0, "f0", {"f1"}, "field f0 does not exist in schema"},
+ {1,
+ "f0",
+ {"f0"},
+ "field f0 has field idx 0 in input schema, mismatch field idx 1 in
predicate"},
+ {0, "f1", {"f1"}, "schema type int32 mismatches predicate field type
STRING"}};
+ for (const auto& test_case : cases) {
+ auto match =
+ std::make_shared<FullTextMatch>(test_case.field_name,
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(auto predicate,
+
PredicateBuilder::FullTextMatch(test_case.field_index, match));
+ ReadContextBuilder builder(path);
+
builder.SetReadFieldNames(test_case.projection).SetPredicate(predicate);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> context,
builder.Finish());
+ if (test_case.expected_error.empty()) {
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<InternalReadContext> internal,
+ InternalReadContext::Create(context, table_schema,
table_schema->Options()));
+ ASSERT_TRUE(internal->GetIndexScoreBindings().empty());
+ ASSERT_FALSE(internal->NeedsScores(
+ {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH,
match->field_name}));
+ ASSERT_NOK_WITH_MSG(
+ InternalReadContext::CreateWithSchema(internal,
internal->GetReadSchema()),
+ "CreateWithSchema does not support");
+ } else {
+ ASSERT_NOK_WITH_MSG(
+ InternalReadContext::Create(context, table_schema,
table_schema->Options()),
+ test_case.expected_error);
+ }
+ }
+}
+
+TEST(InternalReadContext, TestPerFieldTopKScoreBindings) {
+ using SourceType = InternalReadContext::IndexScoreSource::Type;
+ std::string path = paimon::test::GetDataDir() +
"/orc/append_09.db/append_09";
+ SchemaManager schema_manager(std::make_shared<LocalFileSystem>(), path);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
schema_manager.ReadSchema(0));
+ for (bool vector_search : {false, true}) {
+ for (bool project_score : {false, true}) {
+ ReadContextBuilder builder(path);
+ std::vector<std::string> projection = {"f1"};
+ if (project_score) {
+ projection.push_back(vector_search ? "_f3_INDEX_SCORE" :
"_f0_INDEX_SCORE");
+ }
+ builder.SetReadFieldNames(projection);
+ const InternalReadContext::IndexScoreSource source = {
+ vector_search ? SourceType::VECTOR_SEARCH :
SourceType::FULL_TEXT_SEARCH,
+ vector_search ? "f3" : "f0"};
+ if (vector_search) {
+ auto search = std::make_shared<VectorSearch>(
+ "f3", /*limit=*/1, std::vector<float>{1.0f}, nullptr,
nullptr, std::nullopt,
+ std::map<std::string, std::string>{});
+ builder.SetVectorSearch(search);
+ } else {
+ auto search = std::make_shared<FullTextSearch>(
+ "f0", R"({"match":{"query":"apple"}})", /*limit=*/1);
+ builder.SetFullTextSearch(search);
+ }
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> context,
builder.Finish());
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<InternalReadContext> internal,
+ InternalReadContext::Create(context, table_schema,
table_schema->Options()));
+ ASSERT_EQ(internal->GetReadSchema()->field_names(), projection);
+ ASSERT_TRUE(internal->NeedsScores(source));
+ ASSERT_FALSE(internal->NeedsScores(
+ {vector_search ? SourceType::FULL_TEXT_SEARCH :
SourceType::VECTOR_SEARCH,
+ source.field_name}));
+ ASSERT_FALSE(internal->NeedsScores({SourceType::FULL_TEXT_MATCH,
source.field_name}));
+ ASSERT_FALSE(internal->NeedsScores({source.type, "missing"}));
+ const auto& bindings = internal->GetIndexScoreBindings();
+ if (project_score) {
+ const std::map<int32_t, InternalReadContext::IndexScoreSource>
expected = {
+ {1, source}};
+ ASSERT_EQ(bindings, expected);
+ ASSERT_EQ(internal->GetReadSchema()->field(1)->type()->id(),
arrow::Type::FLOAT);
+ } else {
+ ASSERT_TRUE(bindings.empty());
+ }
+ ASSERT_NOK_WITH_MSG(
+ InternalReadContext::CreateWithSchema(internal,
internal->GetReadSchema()),
+ "CreateWithSchema does not support");
+ }
+ }
+}
+
+class InternalReadContextScoreTest : public testing::Test {
+ protected:
+ void SetUp() override {
+ ASSERT_OK_AND_ASSIGN(table_schema_,
+ TableSchema::Create(0,
+
arrow::schema({arrow::field("F1", arrow::utf8()),
+
arrow::field("F2", arrow::utf8())}),
+ {}, {}, {}));
+ }
+
+ Result<std::unique_ptr<InternalReadContext>> Create(ReadContextBuilder*
builder) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> context,
builder->Finish());
+ return InternalReadContext::Create(context, table_schema_,
table_schema_->Options());
+ }
+
+ std::shared_ptr<TableSchema> table_schema_;
+};
+
+TEST_F(InternalReadContextScoreTest, BindsFullTextScoresByProjection) {
+ using SourceType = InternalReadContext::IndexScoreSource::Type;
+ auto first = std::make_shared<FullTextMatch>("F1",
R"({"match":{"query":"apple"}})");
+ auto second = std::make_shared<FullTextMatch>("F2", first->query);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> first_predicate,
+ PredicateBuilder::FullTextMatch(0, first));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> second_predicate,
+ PredicateBuilder::FullTextMatch(1, second));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::And({first_predicate,
second_predicate}));
+ struct TestCase {
+ int32_t score_count;
+ bool explicit_schema;
+ bool reverse_scores;
+ };
+ const std::vector<TestCase> cases = {{0, false, false}, {1, false, false},
{2, false, false},
+ {2, false, true}, {2, true, false},
{2, true, true}};
+ for (const auto& test_case : cases) {
+ const int32_t score_count = test_case.score_count;
+ ReadContextBuilder builder("/tmp/unused-table-path");
+ std::vector<std::string> projection = {"F1", "F2"};
+ if (score_count >= 1) {
+ projection.push_back(test_case.reverse_scores ? "_F1_INDEX_SCORE"
: "_F2_INDEX_SCORE");
+ }
+ if (score_count == 2) {
+ projection.push_back(test_case.reverse_scores ? "_F2_INDEX_SCORE"
: "_F1_INDEX_SCORE");
+ }
+ builder.SetPredicate(predicate);
+ if (test_case.explicit_schema) {
+ arrow::FieldVector fields = {arrow::field("F1", arrow::utf8()),
+ arrow::field("F2", arrow::utf8())};
+ for (int32_t i = 0; i < score_count; ++i) {
+ fields.push_back(arrow::field(projection[i + 2],
arrow::float32()));
+ }
+ auto c_schema = std::make_unique<ArrowSchema>();
+ ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields),
c_schema.get()).ok());
+ builder.SetReadSchema(std::move(c_schema));
+ } else if (score_count > 0) {
+ builder.SetReadFieldNames(projection);
+ }
+ ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder));
+ ASSERT_EQ(internal->GetReadSchema()->field_names(), projection);
+ ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH,
first->field_name}),
+ score_count == 2);
+ ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH,
second->field_name}),
+ score_count >= 1);
+ auto copied_match = std::make_shared<FullTextMatch>(*second);
+ ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH,
copied_match->field_name}),
+ score_count >= 1);
+ ASSERT_FALSE(internal->NeedsScores({SourceType::FULL_TEXT_SEARCH,
second->field_name}));
+ ASSERT_FALSE(internal->NeedsScores({SourceType::FULL_TEXT_MATCH,
"missing"}));
+ const auto& bindings = internal->GetIndexScoreBindings();
+ ASSERT_EQ(bindings.size(), score_count);
+ ASSERT_OK_AND_ASSIGN(std::vector<DataField> fields,
+
DataField::ConvertArrowSchemaToDataFields(internal->GetReadSchema()));
+ for (int32_t i = 0; i < score_count; ++i) {
+ const int32_t output_idx = i + 2;
+ auto iter = bindings.find(output_idx);
+ ASSERT_NE(iter, bindings.end());
+ ASSERT_EQ(iter->second.type, SourceType::FULL_TEXT_MATCH);
+ ASSERT_EQ(iter->second.field_name,
+ projection[output_idx] == "_F1_INDEX_SCORE" ? "F1" :
"F2");
+ ASSERT_EQ(fields[output_idx].Type()->id(), arrow::Type::FLOAT);
+ ASSERT_LT(fields[output_idx].Id(), 0);
+ ASSERT_NE(fields[output_idx].Id(), SpecialFieldIds::INDEX_SCORE);
+ for (int32_t previous = 0; previous < output_idx; ++previous) {
+ ASSERT_NE(fields[output_idx].Id(), fields[previous].Id());
+ }
+ }
+ }
+}
+
+TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) {
+ using SourceType = InternalReadContext::IndexScoreSource::Type;
+ const InternalReadContext::IndexScoreSource search_source =
{SourceType::FULL_TEXT_SEARCH,
+ "F2"};
+ const InternalReadContext::IndexScoreSource match_source =
{SourceType::FULL_TEXT_MATCH, "F1"};
+ auto match = std::make_shared<FullTextMatch>("F1",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::FullTextMatch(0, match));
+ auto search = std::make_shared<FullTextSearch>("F2", match->query,
/*limit=*/10);
+ ReadContextBuilder builder("/tmp/unused-table-path");
+
builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames(
+ {"F1", "_F2_INDEX_SCORE", "_F1_INDEX_SCORE"});
+ ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder));
+ ASSERT_EQ(internal->GetReadSchema()->GetFieldIndex("F2"), -1);
+ const auto& bindings = internal->GetIndexScoreBindings();
+ const std::map<int32_t, InternalReadContext::IndexScoreSource> expected =
{{1, search_source},
+
{2, match_source}};
+ ASSERT_EQ(bindings, expected);
+ ASSERT_TRUE(internal->NeedsScores(search_source));
+ ASSERT_TRUE(internal->NeedsScores(match_source));
+
+
builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames(
+ {"F1", "_F2_INDEX_SCORE", "F2", "_F1_INDEX_SCORE"});
+ ASSERT_OK_AND_ASSIGN(internal, Create(&builder));
+ const std::map<int32_t, InternalReadContext::IndexScoreSource>
non_contiguous = {
+ {1, search_source}, {3, match_source}};
+ ASSERT_EQ(internal->GetIndexScoreBindings(), non_contiguous);
+
+ auto scalar = PredicateBuilder::IsNotNull(0, "F1", FieldType::STRING);
+ ASSERT_OK_AND_ASSIGN(predicate, PredicateBuilder::Or({predicate, scalar}));
+
builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames(
+ {"F1", "_F2_INDEX_SCORE"});
+ ASSERT_OK_AND_ASSIGN(internal, Create(&builder));
+ ASSERT_TRUE(internal->NeedsScores(search_source));
+ ASSERT_FALSE(internal->NeedsScores(match_source));
+ ASSERT_EQ(internal->GetIndexScoreBindings().size(), 1);
+}
+
+TEST_F(InternalReadContextScoreTest,
RejectsAmbiguousOrConditionalScoreProjection) {
+ using SourceType = InternalReadContext::IndexScoreSource::Type;
+ auto match = std::make_shared<FullTextMatch>("F1",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> text,
+ PredicateBuilder::FullTextMatch(0, match));
+ auto scalar = PredicateBuilder::IsNotNull(0, "F1", FieldType::STRING);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> duplicate,
PredicateBuilder::And({text, text}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> conditional,
+ PredicateBuilder::Or({text, scalar}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> scalar_or,
+ PredicateBuilder::Or({scalar, scalar}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> valid,
+ PredicateBuilder::And({text, scalar_or}));
+ struct TestCase {
+ std::shared_ptr<Predicate> predicate;
+ bool root_search;
+ std::string error;
+ };
+ const std::vector<TestCase> cases = {
+ {duplicate, false, "Score-producing search field 'F1' occurs more than
once"},
+ {text, true, "Score-producing search field 'F1' occurs more than
once"},
+ {conditional, false, "Score-producing FullTextMatch cannot appear
under OR"},
+ {valid, false, ""}};
+ for (const auto& test_case : cases) {
+ for (bool project_score : {false, true}) {
+ ReadContextBuilder builder("/tmp/unused-table-path");
+ builder.SetPredicate(test_case.predicate)
+ .SetReadFieldNames(project_score ?
std::vector<std::string>{"F1", "_F1_INDEX_SCORE"}
+ :
std::vector<std::string>{"F1"});
+ if (test_case.root_search) {
+ builder.SetFullTextSearch(
+ std::make_shared<FullTextSearch>("F1", match->query,
/*limit=*/1));
+ }
+ if (project_score && !test_case.error.empty()) {
+ ASSERT_NOK_WITH_MSG(Create(&builder), test_case.error);
+ } else {
+ ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder));
+ ASSERT_EQ(internal->GetIndexScoreBindings().empty(),
!project_score);
+ ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_SEARCH,
"F1"}),
+ test_case.root_search);
+ ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH,
"F1"}),
+ project_score);
+ }
+ }
+ }
+}
+
+TEST_F(InternalReadContextScoreTest, RejectsMissingScoreSource) {
+ for (const auto& name : {"_F1_INDEX_SCORE", "_missing_INDEX_SCORE"}) {
+ ReadContextBuilder builder("/tmp/unused-table-path");
+ builder.SetReadFieldNames({name});
+ ASSERT_NOK_WITH_MSG(Create(&builder),
+ fmt::format("Index score field '{}' has no search
source", name));
+ }
+}
+
+TEST_F(InternalReadContextScoreTest,
ResolvesScoresOnlyFromEffectiveProjection) {
+ const InternalReadContext::IndexScoreSource search_source = {
+ InternalReadContext::IndexScoreSource::Type::FULL_TEXT_SEARCH, "F2"};
+ auto search = std::make_shared<FullTextSearch>("F2",
R"({"match":{"query":"apple"}})", 1);
+ for (bool explicit_schema : {false, true}) {
+ ReadContextBuilder builder("/tmp/unused-table-path");
+ builder.SetFullTextSearch(search)
+ .SetReadFieldNames({"_missing_INDEX_SCORE"})
+ .SetReadFieldIds({0});
+ if (explicit_schema) {
+ auto c_schema = std::make_unique<ArrowSchema>();
+ ASSERT_TRUE(arrow::ExportSchema(
+ *arrow::schema({arrow::field("_F2_INDEX_SCORE",
arrow::float32())}),
+ c_schema.get())
+ .ok());
+ builder.SetReadSchema(std::move(c_schema));
+ }
+ ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder));
+ ASSERT_EQ(internal->GetReadSchema()->field_names(),
+ explicit_schema ? std::vector<std::string>{"_F2_INDEX_SCORE"}
+ : std::vector<std::string>{"F1"});
+ ASSERT_TRUE(internal->NeedsScores(search_source));
+ const auto& bindings = internal->GetIndexScoreBindings();
+ ASSERT_EQ(bindings.size(), explicit_schema ? 1 : 0);
+ if (explicit_schema) {
+ const std::map<int32_t, InternalReadContext::IndexScoreSource>
expected = {
+ {0, search_source}};
+ ASSERT_EQ(bindings, expected);
+ }
+ }
+}
+
+TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) {
+ ReadContextBuilder builder("/tmp/unused-table-path");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<InternalReadContext> internal,
Create(&builder));
+ for (const auto& schema :
+ {arrow::schema({arrow::field("F1", arrow::utf8())}),
+ arrow::schema({arrow::field("_F1_INDEX_SCORE", arrow::float32())})})
{
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<InternalReadContext> replaced,
+ InternalReadContext::CreateWithSchema(internal,
schema));
+ ASSERT_TRUE(replaced->GetReadSchema()->Equals(schema));
+ ASSERT_TRUE(replaced->GetIndexScoreBindings().empty());
+ }
+}
+
TEST(InternalReadContext, TestReadWithValueKindField) {
std::string path = paimon::test::GetDataDir() +
"/orc/append_09.db/append_09";
ReadContextBuilder context_builder(path);
diff --git a/src/paimon/core/operation/read_context.cpp
b/src/paimon/core/operation/read_context.cpp
index 8389ed90..d731c4d3 100644
--- a/src/paimon/core/operation/read_context.cpp
+++ b/src/paimon/core/operation/read_context.cpp
@@ -387,6 +387,24 @@ Result<std::unique_ptr<ReadContext>>
ReadContextBuilder::Finish() {
if (impl_->vector_search_ && impl_->full_text_search_) {
return Status::Invalid("VectorSearch and FullTextSearch cannot be
configured together");
}
+ if (impl_->vector_search_) {
+ const auto& search = impl_->vector_search_;
+ if (search->limit <= 0) {
+ return Status::Invalid("VectorSearch TopK limit must be positive");
+ }
+ if (search->field_name.empty() || search->query.empty()) {
+ return Status::Invalid("VectorSearch field name and query must not
be empty");
+ }
+ }
+ if (impl_->full_text_search_) {
+ const auto& search = impl_->full_text_search_;
+ if (search->limit <= 0) {
+ return Status::Invalid("FullTextSearch TopK limit must be
positive");
+ }
+ if (search->field_name.empty() || search->query.empty()) {
+ return Status::Invalid("FullTextSearch field name and query must
not be empty");
+ }
+ }
if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) {
return Status::Invalid("cannot read with null format table");
}
diff --git a/src/paimon/core/operation/read_context_test.cpp
b/src/paimon/core/operation/read_context_test.cpp
index 35955807..d9079e70 100644
--- a/src/paimon/core/operation/read_context_test.cpp
+++ b/src/paimon/core/operation/read_context_test.cpp
@@ -29,7 +29,9 @@
#include "paimon/defs.h"
#include "paimon/executor.h"
#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/full_text_search.h"
+#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/status.h"
@@ -432,4 +434,39 @@ TEST(ReadContextTest,
TestRejectConflictingFileIndexSearch) {
ASSERT_NOK_WITH_MSG(conflicting.Finish(), "cannot be configured together");
}
+TEST(ReadContextTest, TestFullTextPredicateAndRootTopKConfiguration) {
+ auto match = std::make_shared<FullTextMatch>("body",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> text,
+ PredicateBuilder::FullTextMatch(0, match));
+ auto scalar = PredicateBuilder::LessThan(1, "price", FieldType::INT,
Literal(10));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> filter,
PredicateBuilder::Or({text, scalar}));
+ auto vector =
+ std::make_shared<VectorSearch>("embedding", 3,
std::vector<float>{1.0f}, nullptr, nullptr,
+ std::nullopt, std::map<std::string,
std::string>{});
+ ReadContextBuilder builder("table_root_path");
+ builder.SetReadFieldNames({"body",
"price"}).SetPredicate(filter).SetVectorSearch(vector);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> context,
builder.Finish());
+ ASSERT_EQ(context->GetPredicate(), filter);
+ ASSERT_EQ(context->GetVectorSearch(), vector);
+
+ auto ranked_text = std::make_shared<FullTextSearch>("title", match->query,
2);
+ builder.SetPredicate(filter).SetFullTextSearch(ranked_text);
+ ASSERT_OK_AND_ASSIGN(context, builder.Finish());
+ ASSERT_EQ(context->GetPredicate(), filter);
+ ASSERT_EQ(context->GetFullTextSearch(), ranked_text);
+}
+
+TEST(ReadContextTest, TestRootTopKValidation) {
+ auto vector =
+ std::make_shared<VectorSearch>("embedding", 0,
std::vector<float>{1.0f}, nullptr, nullptr,
+ std::nullopt, std::map<std::string,
std::string>{});
+ ReadContextBuilder vector_builder("table_root_path");
+ vector_builder.SetVectorSearch(vector);
+ ASSERT_NOK_WITH_MSG(vector_builder.Finish(), "VectorSearch TopK limit must
be positive");
+ auto text = std::make_shared<FullTextSearch>("body",
R"({"match":{"query":"apple"}})", -1);
+ ReadContextBuilder text_builder("table_root_path");
+ text_builder.SetFullTextSearch(text);
+ ASSERT_NOK_WITH_MSG(text_builder.Finish(), "FullTextSearch TopK limit must
be positive");
+}
+
} // namespace paimon::test
diff --git a/src/paimon/core/table/source/table_read_test.cpp
b/src/paimon/core/table/source/table_read_test.cpp
index 4f5c03b5..31e3f74f 100644
--- a/src/paimon/core/table/source/table_read_test.cpp
+++ b/src/paimon/core/table/source/table_read_test.cpp
@@ -27,20 +27,26 @@
#include <vector>
#include "gtest/gtest.h"
+#include "paimon/catalog/table.h"
#include "paimon/common/data/binary_row.h"
#include "paimon/core/core_options.h"
#include "paimon/core/operation/abstract_split_read.h"
#include "paimon/core/operation/split_read.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/source/append_only_table_read.h"
#include "paimon/core/table/source/data_split_impl.h"
#include "paimon/core/table/source/key_value_table_read.h"
#include "paimon/defs.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/full_text_search.h"
#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/read_context.h"
#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
#include "paimon/testing/utils/binary_row_generator.h"
#include "paimon/testing/utils/testharness.h"
@@ -117,6 +123,52 @@ TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) {
"Get field field_no_exist failed: not exist in table
schema");
}
+TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) {
+ auto match = std::make_shared<FullTextMatch>("f0",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(auto full_text,
PredicateBuilder::FullTextMatch(/*field_index=*/0, match));
+ auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1",
FieldType::INT, Literal(1));
+ ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar,
full_text}));
+ const std::string table_path = paimon::test::GetDataDir() +
"/orc/append_09.db/append_09";
+ auto fs = std::make_shared<LocalFileSystem>();
+ SchemaManager schema_manager(fs, table_path);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> schema,
schema_manager.ReadSchema(0));
+ Table table(fs, table_path, schema, Identifier("append_09.db",
"append_09"));
+ for (bool table_entry : {false, true}) {
+ ReadContextBuilder builder(table_path);
+ builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate);
+ ASSERT_OK_AND_ASSIGN(auto context, builder.Finish());
+ auto result =
+ table_entry ? table.NewRead(std::move(context)) :
TableRead::Create(std::move(context));
+ ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not
implemented yet");
+ ASSERT_TRUE(result.status().IsNotImplemented());
+
+ builder.SetReadFieldNames({"f0", "f1",
"_f0_INDEX_SCORE"}).SetPredicate(predicate);
+ ASSERT_OK_AND_ASSIGN(context, builder.Finish());
+ result =
+ table_entry ? table.NewRead(std::move(context)) :
TableRead::Create(std::move(context));
+ ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot
appear under OR");
+ }
+}
+
+TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) {
+ const std::string path = paimon::test::GetDataDir() +
"/orc/append_09.db/append_09";
+ auto search =
+ std::make_shared<FullTextSearch>("f0",
R"({"match":{"query":"apple"}})", /*limit=*/1);
+ auto fs = std::make_shared<LocalFileSystem>();
+ SchemaManager schema_manager(fs, path);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> schema,
schema_manager.ReadSchema(0));
+ Table table(fs, path, schema, Identifier("append_09.db", "append_09"));
+ for (bool table_entry : {false, true}) {
+ ReadContextBuilder builder(path);
+ builder.SetFullTextSearch(search).SetReadFieldNames({"f1",
"_f0_INDEX_SCORE"});
+ ASSERT_OK_AND_ASSIGN(auto context, builder.Finish());
+ auto result =
+ table_entry ? table.NewRead(std::move(context)) :
TableRead::Create(std::move(context));
+ ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not
implemented yet");
+ ASSERT_TRUE(result.status().IsNotImplemented());
+ }
+}
+
TEST(TableReadTest, TestCreateKeyValueTableRead) {
std::string path = paimon::test::GetDataDir() +
"/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp
index 944ffab2..05751f92 100644
--- a/src/paimon/core/table/table.cpp
+++ b/src/paimon/core/table/table.cpp
@@ -104,6 +104,7 @@
#include "paimon/file_store_write.h"
#include "paimon/format/file_format.h"
#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate_utils.h"
#include "paimon/read_context.h"
#include "paimon/realtime/realtime_context.h"
#include "paimon/result.h"
@@ -988,6 +989,12 @@ Result<std::unique_ptr<InternalReadContext>>
CreateInternalReadContext(
Result<std::unique_ptr<TableRead>> CreateTableRead(
const std::shared_ptr<InternalReadContext>& internal_context,
const std::shared_ptr<MemoryPool>& memory_pool, const
std::shared_ptr<Executor>& executor) {
+ if
(PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) {
+ return Status::NotImplemented("FullTextMatch predicate execution is
not implemented yet");
+ }
+ if (!internal_context->GetIndexScoreBindings().empty()) {
+ return Status::NotImplemented("Per-field index score output is not
implemented yet");
+ }
const auto& core_options = internal_context->GetCoreOptions();
const auto& table_schema = internal_context->GetTableSchema();
if (internal_context->GetRealtimeContext() &&
!core_options.RealtimeEnabled()) {
@@ -1051,6 +1058,9 @@ Result<std::unique_ptr<TableRead>> NewDataTableRead(
/// refuses by name what a format table cannot honour.
Result<std::unique_ptr<TableRead>> NewFormatTableRead(const
std::shared_ptr<FormatTable>& table,
const
std::shared_ptr<ReadContext>& context) {
+ if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) {
+ return Status::NotImplemented("FullTextMatch predicate execution is
not implemented yet");
+ }
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableRead> read,
FormatTableRead::Create(table, context));
return std::unique_ptr<TableRead>(std::move(read));
@@ -1086,6 +1096,10 @@ Result<std::unique_ptr<TableRead>>
TableRead::Create(std::unique_ptr<ReadContext
PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path,
SystemTableLoader::TryParsePath(context->GetPath()));
if (system_table_path) {
+ if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) {
+ return Status::NotImplemented(
+ "FullTextMatch predicate execution is not implemented yet");
+ }
if (context->HasVectorOrFullTextSearch()) {
return Status::NotImplemented(
"VectorSearch or FullTextSearch is not supported for system
tables");
diff --git a/src/paimon/core/utils/field_mapping.cpp
b/src/paimon/core/utils/field_mapping.cpp
index 70668a5b..7155c16f 100644
--- a/src/paimon/core/utils/field_mapping.cpp
+++ b/src/paimon/core/utils/field_mapping.cpp
@@ -25,6 +25,7 @@
#include "arrow/type.h"
#include "fmt/format.h"
#include "paimon/common/predicate/compound_predicate_impl.h"
+#include "paimon/common/predicate/full_text_match_predicate.h"
#include "paimon/common/predicate/leaf_predicate_impl.h"
#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/field_type_utils.h"
@@ -344,9 +345,8 @@ Result<std::optional<std::shared_ptr<Predicate>>>
FieldMappingBuilder::Reconstru
const auto& [data_field_idx, data_field] = data_field_iter->second;
return
std::optional<std::shared_ptr<Predicate>>(std::make_shared<LeafPredicateImpl>(
- leaf_predicate->GetLeafFunction(), /*data field idx*/ data_field_idx,
- /*data field name*/ data_field.Name(),
- /*data field type*/ data_type, new_literals));
+ leaf_predicate->GetLeafFunction(), /*data_field_idx=*/data_field_idx,
+ /*data_field_name=*/data_field.Name(), /*data_field_type=*/data_type,
new_literals));
}
Result<std::optional<std::shared_ptr<Predicate>>>
@@ -354,7 +354,25 @@ FieldMappingBuilder::ReconstructPredicateWithDataFields(
const std::shared_ptr<Predicate>& predicate,
const std::map<std::string, DataField>& field_name_to_read_fields,
const std::map<int32_t, std::pair<int32_t, DataField>>&
field_id_to_data_fields) {
- if (auto leaf_predicate =
std::dynamic_pointer_cast<LeafPredicateImpl>(predicate)) {
+ if (auto full_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate)) {
+ auto read_field =
field_name_to_read_fields.find(full_text->FieldName());
+ if (read_field == field_name_to_read_fields.end()) {
+ return Status::Invalid(
+ fmt::format("FullTextMatch field '{}' is not projected",
full_text->FieldName()));
+ }
+ auto data_field =
field_id_to_data_fields.find(read_field->second.Id());
+ if (data_field == field_id_to_data_fields.end()) {
+ return Status::Invalid(fmt::format(
+ "FullTextMatch field '{}' does not exist in file schema",
full_text->FieldName()));
+ }
+ const auto& [field_index, field] = data_field->second;
+ if (field.Type()->id() != arrow::Type::STRING) {
+ return Status::Invalid(
+ fmt::format("FullTextMatch field '{}' must have STRING type",
field.Name()));
+ }
+ return std::optional<std::shared_ptr<Predicate>>(
+
full_text->NewLeafPredicate(field.Name())->NewLeafPredicate(field_index));
+ } else if (auto leaf_predicate =
std::dynamic_pointer_cast<LeafPredicateImpl>(predicate)) {
return ReconstructLeafPredicate(leaf_predicate,
field_name_to_read_fields,
field_id_to_data_fields);
} else if (auto compound_predicate =
diff --git a/src/paimon/core/utils/field_mapping_test.cpp
b/src/paimon/core/utils/field_mapping_test.cpp
index dbc1d932..4f66cd10 100644
--- a/src/paimon/core/utils/field_mapping_test.cpp
+++ b/src/paimon/core/utils/field_mapping_test.cpp
@@ -24,10 +24,12 @@
#include "arrow/type_fwd.h"
#include "gtest/gtest.h"
+#include "paimon/common/predicate/full_text_match_predicate.h"
#include "paimon/common/predicate/leaf_predicate_impl.h"
#include "paimon/common/predicate/predicate_filter.h"
#include "paimon/data/decimal.h"
#include "paimon/defs.h"
+#include "paimon/predicate/full_text_match.h"
#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/status.h"
@@ -99,6 +101,47 @@ TEST_F(FieldMappingTest,
TestFieldNameMappingUsesIdsAcrossRenameAndReorder) {
ASSERT_TRUE(FieldMappingBuilder::CreateFieldNameMapping(table_fields,
{}).empty());
}
+TEST_F(FieldMappingTest, TestFullTextMatchRenameAndReorder) {
+ RoaringBitmap64 candidates;
+ candidates.Add(3);
+ candidates.Add(9);
+ auto match =
+ std::make_shared<FullTextMatch>("text",
R"({"match":{"query":"apple"}})", candidates);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::FullTextMatch(1, match));
+ std::map<std::string, DataField> read_fields = {
+ {"text", DataField(7, arrow::field("text", arrow::utf8()))}};
+ std::map<int32_t, std::pair<int32_t, DataField>> data_fields = {
+ {7, {3, DataField(7, arrow::field("old_text", arrow::utf8()))}}};
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<Predicate>>
reconstructed,
+
FieldMappingBuilder::ReconstructPredicateWithDataFields(
+ predicate, read_fields, data_fields));
+ ASSERT_TRUE(reconstructed);
+ auto full_text =
std::dynamic_pointer_cast<FullTextMatchPredicate>(reconstructed.value());
+ ASSERT_TRUE(full_text);
+ ASSERT_EQ(full_text->FieldName(), "old_text");
+ ASSERT_EQ(full_text->FieldIndex(), 3);
+ ASSERT_EQ(full_text->GetFullTextMatch()->field_name, "old_text");
+ ASSERT_EQ(full_text->GetFullTextMatch()->query, match->query);
+ ASSERT_EQ(full_text->GetFullTextMatch()->include_row_ids,
match->include_row_ids);
+ ASSERT_NE(full_text->GetFullTextMatch(), match);
+ ASSERT_EQ(match->field_name, "text");
+ auto original =
std::dynamic_pointer_cast<FullTextMatchPredicate>(predicate);
+ ASSERT_EQ(original->FieldName(), "text");
+ ASSERT_EQ(original->FieldIndex(), 1);
+
+ ASSERT_NOK_WITH_MSG(
+ FieldMappingBuilder::ReconstructPredicateWithDataFields(predicate, {},
data_fields),
+ "FullTextMatch field 'text' is not projected");
+ ASSERT_NOK_WITH_MSG(
+ FieldMappingBuilder::ReconstructPredicateWithDataFields(predicate,
read_fields, {}),
+ "FullTextMatch field 'text' does not exist in file schema");
+ data_fields = {{7, {0, DataField(7, arrow::field("old_text",
arrow::int32()))}}};
+
ASSERT_NOK_WITH_MSG(FieldMappingBuilder::ReconstructPredicateWithDataFields(
+ predicate, read_fields, data_fields),
+ "FullTextMatch field 'old_text' must have STRING
type");
+}
+
TEST_F(FieldMappingTest, TestEmptyPartitionKeys) {
// test no partition keys in schema
std::string val("hello");
diff --git a/test/inte/predicate_abi_inte_test.cpp
b/test/inte/predicate_abi_inte_test.cpp
index 3590eb22..02340059 100644
--- a/test/inte/predicate_abi_inte_test.cpp
+++ b/test/inte/predicate_abi_inte_test.cpp
@@ -46,11 +46,15 @@
#include "gtest/gtest.h"
#include "paimon/defs.h"
+#include "paimon/file_index/file_index_reader.h"
#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/full_text_match.h"
+#include "paimon/predicate/function.h"
#include "paimon/predicate/leaf_predicate.h"
#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate.h"
#include "paimon/predicate/predicate_builder.h"
+#include "paimon/predicate/predicate_utils.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
@@ -74,6 +78,23 @@ TEST(PredicateAbiInteTest, LeafPredicateCastsAcrossModule) {
EXPECT_EQ(std::dynamic_pointer_cast<CompoundPredicate>(predicate),
nullptr);
}
+TEST(PredicateAbiInteTest, FullTextMatchCastsToLeafAcrossModule) {
+ auto match = std::make_shared<FullTextMatch>("body",
R"({"match":{"query":"apple"}})");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+ PredicateBuilder::FullTextMatch(0, match));
+ auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate);
+ ASSERT_TRUE(leaf);
+ ASSERT_EQ(leaf->FieldIndex(), 0);
+ ASSERT_EQ(leaf->FieldName(), "body");
+ ASSERT_EQ(leaf->GetFieldType(), FieldType::STRING);
+ ASSERT_TRUE(leaf->Literals().empty());
+ ASSERT_EQ(leaf->GetFunction().GetType(), Function::Type::FULL_TEXT_MATCH);
+ auto reader = std::make_shared<FileIndexReader>();
+ auto result =
PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(leaf, reader);
+ ASSERT_NOK_WITH_MSG(result, "This file index does not support complete
full-text matching");
+ ASSERT_TRUE(result.status().IsNotImplemented());
+}
+
// A compound predicate built in libpaimon.so must survive dynamic_pointer_cast
// to CompoundPredicate, and its children must remain castable to
LeafPredicate.
TEST(PredicateAbiInteTest, CompoundPredicateCastsAcrossModule) {
