zhouyc-ali commented on code in PR #471:
URL: https://github.com/apache/paimon-cpp/pull/471#discussion_r4465889294


##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -262,14 +144,22 @@ Result<std::unique_ptr<VortexFileBatchReader>> 
VortexFileBatchReader::Create(
     auto length = static_cast<uint64_t>(signed_length);
     auto input_context = std::make_shared<VortexInputContext>(input);
 
-    VxSessionPtr session(vx_session_new(), vx_session_free);
+    VxSessionPtr session(vx_paimon_read_session_new(), vx_session_free);
     if (session == nullptr) {
         return Status::IOError("failed to create Vortex session");
     }
     vx_error* error = nullptr;
+    const bool local_coalesce = coalesce_mode == VortexIoCoalesceMode::LOCAL;
+    // Smaller gaps limit read amplification for explicitly selected 
low-latency sources.
+    constexpr uint64_t kLocalMaxGap = 64ULL << 10;
+    constexpr uint64_t kLocalMaxSize = 4ULL << 20;

Review Comment:
   Moved the I/O defaults into `vortex_defs.h` and removed the function-local 
constants.



##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -262,14 +144,22 @@ Result<std::unique_ptr<VortexFileBatchReader>> 
VortexFileBatchReader::Create(
     auto length = static_cast<uint64_t>(signed_length);
     auto input_context = std::make_shared<VortexInputContext>(input);
 
-    VxSessionPtr session(vx_session_new(), vx_session_free);
+    VxSessionPtr session(vx_paimon_read_session_new(), vx_session_free);
     if (session == nullptr) {
         return Status::IOError("failed to create Vortex session");
     }
     vx_error* error = nullptr;
+    const bool local_coalesce = coalesce_mode == VortexIoCoalesceMode::LOCAL;
+    // Smaller gaps limit read amplification for explicitly selected 
low-latency sources.
+    constexpr uint64_t kLocalMaxGap = 64ULL << 10;
+    constexpr uint64_t kLocalMaxSize = 4ULL << 20;
     VxDataSourcePtr data_source(
-        vx_data_source_new_callback(session.get(), 
VortexInputContext::MakeCallbacks(input_context),
-                                    length, &error),
+        local_coalesce
+            ? vx_data_source_new_callback_with_coalesce(
+                  session.get(), 
VortexInputContext::MakeCallbacks(input_context), length,
+                  kLocalMaxGap, kLocalMaxSize, &error)
+            : vx_data_source_new_callback(
+                  session.get(), 
VortexInputContext::MakeCallbacks(input_context), length, &error),

Review Comment:
   Removed `vx_data_source_new_callback`; all callers now use 
`vx_data_source_new_callback_with_coalesce` with explicit limits.



##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -306,20 +193,76 @@ Result<std::unique_ptr<VortexFileBatchReader>> 
VortexFileBatchReader::Create(
     // scan may be consumed only once.
     return std::unique_ptr<VortexFileBatchReader>(new VortexFileBatchReader(
         input, batch_size, std::move(input_context), std::move(session), 
std::move(data_source),
-        VxScanPtr(nullptr, vx_scan_free), file_schema, struct_type, 
total_rows, arrow_pool));
+        VxScanPtr(nullptr, vx_scan_free), file_schema, total_rows, arrow_pool, 
predicate_options));
 }
 
 VortexFileBatchReader::~VortexFileBatchReader() {
     CloseInternal();
 }
 
 Result<bool> VortexFileBatchReader::OpenNextPartitionStream() {
+    if (has_selection_ && selected_row_count_ == 0) {
+        return false;
+    }
     if (scan_ == nullptr) {
         vx_error* error = nullptr;
         // Scan in storage order: Vortex defaults to ordered=false and may 
emit chunks out of order,
         // which would misalign the physical row positions NextBatch assigns.
-        vx_scan_options options{};  // zero-init: all columns, no 
filter/row-range/selection/limit
+        vx_scan_options options{};
         options.ordered = true;
+        // Missing fields are null-filled by AlignToReadSchema and cannot be 
selected here.
+        // The scan clones the projection, so these expressions need only 
outlive scan creation.
+        VxExpressionPtr root(nullptr, vx_expression_free);
+        VxExpressionPtr projection(nullptr, vx_expression_free);
+        std::vector<std::string> names;
+        std::vector<const char*> name_ptrs;
+        if (read_schema_ != nullptr) {
+            names.reserve(read_schema_->num_fields());
+            // Native merge requires distinct fields; output alignment 
restores repeated columns.
+            std::vector<uint8_t> selected_columns(file_schema_->num_fields(), 
0);
+            for (const std::shared_ptr<arrow::Field>& field : 
read_schema_->fields()) {
+                const int32_t field_index = 
file_schema_->GetFieldIndex(field->name());
+                if (field_index >= 0 && selected_columns[field_index] == 0) {
+                    names.push_back(field->name());
+                    selected_columns[field_index] = 1;

Review Comment:
   `SetReadSchema()` now rejects duplicate or missing fields, including nested 
projections, before changing reader state.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to