zhouyc-ali commented on code in PR #471:
URL: https://github.com/apache/paimon-cpp/pull/471#discussion_r4465891964


##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -306,20 +193,76 @@ Result<std::unique_ptr<VortexFileBatchReader>> 
VortexFileBatchReader::Create(
     // scan may be consumed only once.
     return std::unique_ptr<VortexFileBatchReader>(new VortexFileBatchReader(
         input, batch_size, std::move(input_context), std::move(session), 
std::move(data_source),
-        VxScanPtr(nullptr, vx_scan_free), file_schema, struct_type, 
total_rows, arrow_pool));
+        VxScanPtr(nullptr, vx_scan_free), file_schema, total_rows, arrow_pool, 
predicate_options));
 }
 
 VortexFileBatchReader::~VortexFileBatchReader() {
     CloseInternal();
 }
 
 Result<bool> VortexFileBatchReader::OpenNextPartitionStream() {
+    if (has_selection_ && selected_row_count_ == 0) {
+        return false;
+    }
     if (scan_ == nullptr) {
         vx_error* error = nullptr;
         // Scan in storage order: Vortex defaults to ordered=false and may 
emit chunks out of order,
         // which would misalign the physical row positions NextBatch assigns.
-        vx_scan_options options{};  // zero-init: all columns, no 
filter/row-range/selection/limit
+        vx_scan_options options{};
         options.ordered = true;
+        // Missing fields are null-filled by AlignToReadSchema and cannot be 
selected here.
+        // The scan clones the projection, so these expressions need only 
outlive scan creation.
+        VxExpressionPtr root(nullptr, vx_expression_free);
+        VxExpressionPtr projection(nullptr, vx_expression_free);
+        std::vector<std::string> names;
+        std::vector<const char*> name_ptrs;
+        if (read_schema_ != nullptr) {
+            names.reserve(read_schema_->num_fields());
+            // Native merge requires distinct fields; output alignment 
restores repeated columns.
+            std::vector<uint8_t> selected_columns(file_schema_->num_fields(), 
0);
+            for (const std::shared_ptr<arrow::Field>& field : 
read_schema_->fields()) {
+                const int32_t field_index = 
file_schema_->GetFieldIndex(field->name());
+                if (field_index >= 0 && selected_columns[field_index] == 0) {
+                    names.push_back(field->name());
+                    selected_columns[field_index] = 1;
+                }
+            }
+            if (!names.empty()) {
+                name_ptrs.reserve(names.size());
+                for (const std::string& name : names) {
+                    name_ptrs.push_back(name.c_str());
+                }
+                root = VxExpressionPtr(vx_expression_root(), 
vx_expression_free);
+                projection = VxExpressionPtr(
+                    vx_expression_select(name_ptrs.data(), name_ptrs.size(), 
root.get()),
+                    vx_expression_free);
+                options.projection = projection.get();
+            }
+        }
+        // These FFI modes take individual indices. A dense bitmap sends only 
excluded rows;
+        // an all-selected bitmap needs no indices and uses the default 
INCLUDE_ALL mode.
+        if (has_selection_ && !selection_indices_.empty()) {
+            options.selection.idx = selection_indices_.data();
+            options.selection.idx_len = selection_indices_.size();
+            options.selection.include =
+                selection_excludes_ ? VX_SELECTION_EXCLUDE_RANGE : 
VX_SELECTION_INCLUDE_RANGE;
+        }
+        // Filtering drops rows; include_row_idx preserves their original 
physical positions.
+        // The scan clones the filter, so it can be freed after scan creation.
+        VxExpressionPtr filter(nullptr, vx_expression_free);
+        // The hidden index must not collide with a file field or a requested 
missing field.
+        const bool has_row_idx_column =
+            file_schema_->GetFieldIndex(kRowIdxColumn) >= 0 ||
+            (read_schema_ != nullptr && 
!read_schema_->GetAllFieldIndices(kRowIdxColumn).empty());
+        if (!has_row_idx_column) {

Review Comment:
   With schema validation in place, projection now uses read field names 
directly instead of copying them.



##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########


Review Comment:
   Extracted scan creation into `CreateScan()`, leaving partition and stream 
advancement in `OpenNextPartitionStream()`.



##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -448,24 +409,68 @@ Result<std::unique_ptr<::ArrowSchema>> 
VortexFileBatchReader::GetFileSchema() co
 Status VortexFileBatchReader::SetReadSchema(
     ::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
     const std::optional<RoaringBitmap32>& selection_bitmap) {
+    if (closed_) {
+        return Status::Invalid("Vortex reader is closed");
+    }
     if (read_schema == nullptr) {
         return Status::Invalid("Vortex read schema is nullptr");
     }
-    // TODO(zhouyc-ali): push the row selection into the Vortex scan
-    (void)selection_bitmap;
-    // No pushdown: the whole file is scanned and reading restarts from row 0, 
so drop the
-    // single-use scan; the next read re-creates it.
+    std::vector<uint64_t> selection_indices;
+    bool selection_excludes = false;
+    uint64_t selected_row_count = total_rows_;
+    if (selection_bitmap) {
+        const int32_t cardinality = selection_bitmap->Cardinality();
+        if (cardinality < 0 || static_cast<uint64_t>(cardinality) > 
total_rows_) {
+            return Status::Invalid("Vortex selection cardinality exceeds the 
file row count");
+        }
+        selected_row_count = static_cast<uint64_t>(cardinality);
+        selection_excludes = selected_row_count > total_rows_ / 2;
+        selection_indices.reserve(static_cast<size_t>(
+            selection_excludes ? total_rows_ - selected_row_count : 
selected_row_count));
+        uint64_t next_row = 0;
+        const auto end = selection_bitmap->End();
+        for (auto iter = selection_bitmap->Begin(); iter != end; ++iter) {
+            const auto row_id = static_cast<uint64_t>(*iter);
+            if (row_id >= total_rows_) {
+                return Status::Invalid(
+                    fmt::format("Vortex selected row {} is out of range {}", 
row_id, total_rows_));
+            }
+            if (selection_excludes) {
+                // Enumerate gaps directly instead of first expanding all 
selected rows.
+                for (; next_row < row_id; ++next_row) {
+                    selection_indices.push_back(next_row);
+                }
+                next_row = row_id + 1;
+            } else {
+                selection_indices.push_back(row_id);
+            }
+        }
+        if (selection_excludes) {
+            for (; next_row < total_rows_; ++next_row) {
+                selection_indices.push_back(next_row);
+            }
+        }
+    }
+    // Leave the active scan and its row mapping unchanged if validation or 
import fails.
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> 
imported_read_schema,
                                       arrow::ImportSchema(read_schema));
+    // Recreate the single-use scan on the next read with the new projection 
and predicates.
     read_schema_ = std::move(imported_read_schema);
     predicate_filter_ = std::dynamic_pointer_cast<PredicateFilter>(predicate);
+    has_selection_ = selection_bitmap.has_value();
+    selection_excludes_ = selection_excludes;
+    selected_row_count_ = selected_row_count;
+    selection_indices_ = std::move(selection_indices);
+    metrics_->SetCounter("vortex.selection-index-count", 
selection_indices_.size());
     ReleaseStream();

Review Comment:
   Centralized metric names in `vortex_defs.h` and renamed the counter to 
`vortex.selection-row-count`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to