zhouyc-ali commented on code in PR #471:
URL: https://github.com/apache/paimon-cpp/pull/471#discussion_r4465896252


##########
test/inte/scan_and_read_inte_test.cpp:
##########
@@ -3727,6 +3727,233 @@ TEST_F(ScanAndReadInteTest, 
TestMosaicJavaAndPythonCompatibility) {
 #endif
 
 #ifdef PAIMON_ENABLE_VORTEX
+namespace {
+
+Result<std::shared_ptr<arrow::ChunkedArray>> ReadVortexSdkRows(
+    const std::string& table_path, const std::vector<std::shared_ptr<Split>>& 
splits,
+    const std::vector<std::string>& projection, const 
std::shared_ptr<Predicate>& predicate,
+    bool exact_filter, bool late_materializing) {
+    ReadContextBuilder builder(table_path);
+    builder.SetReadFieldNames(projection)
+        .SetPredicate(predicate)
+        .EnablePredicateFilter(exact_filter)
+        .EnableLateMaterializing(late_materializing)
+        .EnablePrefetch(true)
+        .SetPrefetchBatchCount(2)
+        .SetPrefetchMaxParallelNum(2)
+        .AddOption(Options::READ_BATCH_SIZE, "2");
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> context, 
builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> read, 
TableRead::Create(std::move(context)));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, 
read->CreateReader(splits));
+    return ReadResultCollector::CollectResult(std::move(reader));
+}
+
+// Append splits need not have a stable file order. Compare full rows as a 
multiset.
+void AssertVortexSdkRows(const std::shared_ptr<arrow::ChunkedArray>& actual,
+                         const std::shared_ptr<arrow::DataType>& type,
+                         const std::string& expected_json, bool exact_filter = 
true) {
+    auto expected = arrow::ipc::internal::json::ArrayFromJSON(type, 
expected_json).ValueOrDie();
+    if (!actual) {
+        ASSERT_EQ(expected->length(), 0);
+        return;
+    }
+    ASSERT_TRUE(actual->type()->Equals(type)) << actual->type()->ToString();
+    if (exact_filter) {
+        ASSERT_EQ(actual->length(), expected->length()) << actual->ToString();
+    } else {
+        ASSERT_GE(actual->length(), expected->length());
+    }
+    arrow::ArrayVector candidates;
+    for (const auto& chunk : actual->chunks()) {
+        ASSERT_TRUE(chunk->ValidateFull().ok());
+        for (int64_t row = 0; row < chunk->length(); ++row) {
+            candidates.push_back(chunk->Slice(row, 1));
+        }
+    }
+    std::vector<bool> used(candidates.size(), false);
+    for (int64_t row = 0; row < expected->length(); ++row) {
+        bool found = false;
+        for (size_t candidate = 0; candidate < candidates.size(); ++candidate) 
{
+            if (!used[candidate] && expected->Slice(row, 
1)->Equals(candidates[candidate])) {
+                used[candidate] = true;
+                found = true;
+                break;
+            }
+        }
+        ASSERT_TRUE(found) << "Missing row: " << expected->Slice(row, 
1)->ToString()
+                           << "\nActual: " << actual->ToString();
+    }
+}
+
+}  // namespace
+
+TEST_F(ScanAndReadInteTest, 
TestVortexSdkPredicatesWithResidualAndLateMaterialization) {
+    auto dir = UniqueTestDirectory::Create("local");
+    ASSERT_NE(dir, nullptr);
+    const arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
+                                       arrow::field("tag", arrow::utf8()),
+                                       arrow::field("payload", arrow::utf8())};
+    const std::map<std::string, std::string> options = {{Options::FILE_FORMAT, 
"vortex"},
+                                                        {Options::FILE_SYSTEM, 
"local"},
+                                                        {Options::BUCKET, 
"-1"},
+                                                        {Options::WRITE_ONLY, 
"true"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TestHelper> helper,
+                         TestHelper::Create(dir->Str(), arrow::schema(fields), 
{}, {}, options,
+                                            /*is_streaming_mode=*/false));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), 
R"([
+            
[0,"skip","zero"],[1,null,"one"],[2,"keep","two"],[3,"drop","three"],
+            
[4,"keep",null],[5,"keep","five"],[6,"skip","six"],[7,"keep","seven"]
+        ])",
+                                                     {}, 0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt));
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), 
"foo.db/bar");
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> splits,
+                         helper->NewScan(StartupMode::LatestFull(), 
std::nullopt, false));
+    const auto result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), fields[2], fields[0], 
fields[1]});
+    auto equal = PredicateBuilder::Equal(1, "id", FieldType::INT, 
Literal(int32_t{4}));
+    auto greater = PredicateBuilder::GreaterThan(1, "id", FieldType::INT, 
Literal(int32_t{1}));
+    auto tag =
+        PredicateBuilder::Equal(2, "tag", FieldType::STRING, 
Literal(FieldType::STRING, "keep", 4));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> conjunction,
+                         PredicateBuilder::And({greater, tag}));
+    // Sparse IN(64) exceeds the auto expression budget and requires the SDK 
residual filter.
+    std::vector<Literal> sparse = {Literal(int32_t{2}), Literal(int32_t{5})};
+    for (int32_t index = 0; index < 62; ++index) {
+        sparse.emplace_back(1000 + index * 2);
+    }
+    auto membership = PredicateBuilder::In(1, "id", FieldType::INT, sparse);
+    const std::vector<std::pair<std::shared_ptr<Predicate>, std::string>> 
cases = {
+        {equal, R"([[0,null,4,"keep"]])"},
+        {conjunction, R"([[0,"two",2,"keep"],[0,null,4,"keep"],
+                           [0,"five",5,"keep"],[0,"seven",7,"keep"]])"},
+        {membership, R"([[0,"two",2,"keep"],[0,"five",5,"keep"]])"}};
+    for (bool late_materializing : {false, true}) {
+        SCOPED_TRACE(late_materializing);
+        for (const auto& [predicate, expected] : cases) {
+            SCOPED_TRACE(predicate->ToString());
+            for (bool exact_filter : {false, true}) {
+                SCOPED_TRACE(exact_filter);
+                ASSERT_OK_AND_ASSIGN(
+                    std::shared_ptr<arrow::ChunkedArray> actual,
+                    ReadVortexSdkRows(table_path, splits, {"payload", "id", 
"tag"}, predicate,
+                                      exact_filter, late_materializing));
+                // A non-exact reader may prune candidates, but it must retain 
every matching row.
+                ASSERT_NO_FATAL_FAILURE(
+                    AssertVortexSdkRows(actual, result_type, expected, 
exact_filter));
+            }

Review Comment:
   Updated this append test to require exact results when either mode is 
enabled, since every case has both probe and payload columns. The shared read 
path is unchanged.



##########
src/paimon/format/vortex/vortex_predicate_converter.h:
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+
+#include "paimon/format/vortex/vortex_ffi_util.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+class Predicate;
+}  // namespace paimon
+
+namespace paimon::vortex {
+
+enum class MembershipPushdownMode { AUTO, FORCE, OFF };
+
+struct VortexPredicateOptions {
+    MembershipPushdownMode membership_pushdown = MembershipPushdownMode::AUTO;
+    // Per-leaf comparison and boolean node budget after deduplication/range 
compression.
+    uint32_t max_membership_nodes = 64;
+};
+
+/// Converts Paimon predicates into Vortex scan filters, leaving unsupported 
conditions to the
+/// caller's residual filter.
+class VortexPredicateConverter {
+ public:
+    VortexPredicateConverter() = delete;
+    ~VortexPredicateConverter() = delete;
+
+    /// @param predicate The predicate to convert.
+    /// @param file_schema Stored column types, including timestamp units and 
zones.
+    /// @param options Membership pushdown policy and expression budget.
+    /// @return An owned expression, or nullptr if no condition can be pushed.
+    static VxExpressionPtr Convert(const std::shared_ptr<Predicate>& predicate,
+                                   const std::shared_ptr<arrow::Schema>& 
file_schema,
+                                   const VortexPredicateOptions& options = {});

Review Comment:
   Removed `Convert()`'s default argument; callers already pass the options 
explicitly.



##########
src/paimon/format/vortex/vortex_predicate_converter.cpp:
##########
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/format/vortex/vortex_predicate_converter.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <limits>
+#include <string>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "arrow/type.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/function.h"
+#include "paimon/predicate/leaf_predicate.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+
+namespace paimon::vortex {
+namespace {
+
+VxExpressionPtr NullExpr() {
+    return VxExpressionPtr(nullptr, vx_expression_free);
+}
+
+void ClearError(vx_error* error) {
+    if (error != nullptr) {
+        vx_error_free(error);
+    }
+}
+
+// vx_expression_not returns an owned handle typed const; reclaim it as an 
owned non-const pointer.
+VxExpressionPtr MakeNotExpr(const vx_expression* child) {
+    return 
VxExpressionPtr(const_cast<vx_expression*>(vx_expression_not(child)),
+                           vx_expression_free);
+}
+
+// Reject precision loss and overflow before constructing a native timestamp 
scalar.
+bool GetTimestampValue(const Literal& literal, const 
std::shared_ptr<arrow::DataType>& arrow_type,
+                       int64_t* result, vx_time_unit* result_unit) {
+    if (arrow_type == nullptr || arrow_type->id() != arrow::Type::TIMESTAMP) {
+        return false;
+    }
+    const auto& timestamp_type = checked_cast<const 
arrow::TimestampType&>(*arrow_type);
+    if (!timestamp_type.timezone().empty() && timestamp_type.timezone() != 
"UTC") {
+        return false;
+    }
+    const auto timestamp = literal.GetValue<Timestamp>();

Review Comment:
   New LTZ files use UTC metadata without changing epoch values; C++/Java 
compatibility tests pass. Legacy non-UTC predicates are not pushed down in C++.



##########
src/paimon/format/vortex/vortex_predicate_converter.cpp:
##########
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/format/vortex/vortex_predicate_converter.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <limits>
+#include <string>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "arrow/type.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/function.h"
+#include "paimon/predicate/leaf_predicate.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+
+namespace paimon::vortex {
+namespace {
+
+VxExpressionPtr NullExpr() {
+    return VxExpressionPtr(nullptr, vx_expression_free);
+}
+
+void ClearError(vx_error* error) {
+    if (error != nullptr) {
+        vx_error_free(error);
+    }
+}
+
+// vx_expression_not returns an owned handle typed const; reclaim it as an 
owned non-const pointer.
+VxExpressionPtr MakeNotExpr(const vx_expression* child) {
+    return 
VxExpressionPtr(const_cast<vx_expression*>(vx_expression_not(child)),
+                           vx_expression_free);
+}
+
+// Reject precision loss and overflow before constructing a native timestamp 
scalar.
+bool GetTimestampValue(const Literal& literal, const 
std::shared_ptr<arrow::DataType>& arrow_type,
+                       int64_t* result, vx_time_unit* result_unit) {
+    if (arrow_type == nullptr || arrow_type->id() != arrow::Type::TIMESTAMP) {
+        return false;
+    }
+    const auto& timestamp_type = checked_cast<const 
arrow::TimestampType&>(*arrow_type);
+    if (!timestamp_type.timezone().empty() && timestamp_type.timezone() != 
"UTC") {
+        return false;
+    }
+    const auto timestamp = literal.GetValue<Timestamp>();
+    int64_t value = timestamp.GetMillisecond();
+    const int32_t nanos = timestamp.GetNanoOfMillisecond();
+    vx_time_unit unit = VX_TIME_UNIT_MILLISECOND;
+    int64_t factor = 1;
+    int64_t fraction = 0;
+    switch (timestamp_type.unit()) {
+        case arrow::TimeUnit::SECOND:
+            if (value % 1000 != 0 || nanos != 0) {
+                return false;
+            }
+            value /= 1000;
+            unit = VX_TIME_UNIT_SECOND;
+            break;
+        case arrow::TimeUnit::MILLI:
+            if (nanos != 0) {
+                return false;
+            }
+            break;
+        case arrow::TimeUnit::MICRO:
+            if (nanos % 1000 != 0) {
+                return false;
+            }
+            factor = 1000;
+            fraction = nanos / 1000;
+            unit = VX_TIME_UNIT_MICROSECOND;
+            break;
+        case arrow::TimeUnit::NANO:
+            factor = 1000000;
+            fraction = nanos;
+            unit = VX_TIME_UNIT_NANOSECOND;
+            break;
+    }
+    if (value < std::numeric_limits<int64_t>::min() / factor ||
+        value > std::numeric_limits<int64_t>::max() / factor) {
+        return false;
+    }
+    value *= factor;
+    if (value > std::numeric_limits<int64_t>::max() - fraction) {
+        return false;
+    }
+    *result = value + fraction;
+    *result_unit = unit;
+    return true;
+}

Review Comment:
   Reused `DateTimeUtils::TimestampToInteger()` and 
`GetTimeTypeFromArrowType()`, retaining overflow and lossless-conversion checks.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to