zhouyc-ali commented on code in PR #471:
URL: https://github.com/apache/paimon-cpp/pull/471#discussion_r4465891964
##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -306,20 +193,76 @@ Result<std::unique_ptr<VortexFileBatchReader>>
VortexFileBatchReader::Create(
// scan may be consumed only once.
return std::unique_ptr<VortexFileBatchReader>(new VortexFileBatchReader(
input, batch_size, std::move(input_context), std::move(session),
std::move(data_source),
- VxScanPtr(nullptr, vx_scan_free), file_schema, struct_type,
total_rows, arrow_pool));
+ VxScanPtr(nullptr, vx_scan_free), file_schema, total_rows, arrow_pool,
predicate_options));
}
VortexFileBatchReader::~VortexFileBatchReader() {
CloseInternal();
}
Result<bool> VortexFileBatchReader::OpenNextPartitionStream() {
+ if (has_selection_ && selected_row_count_ == 0) {
+ return false;
+ }
if (scan_ == nullptr) {
vx_error* error = nullptr;
// Scan in storage order: Vortex defaults to ordered=false and may
emit chunks out of order,
// which would misalign the physical row positions NextBatch assigns.
- vx_scan_options options{}; // zero-init: all columns, no
filter/row-range/selection/limit
+ vx_scan_options options{};
options.ordered = true;
+ // Missing fields are null-filled by AlignToReadSchema and cannot be
selected here.
+ // The scan clones the projection, so these expressions need only
outlive scan creation.
+ VxExpressionPtr root(nullptr, vx_expression_free);
+ VxExpressionPtr projection(nullptr, vx_expression_free);
+ std::vector<std::string> names;
+ std::vector<const char*> name_ptrs;
+ if (read_schema_ != nullptr) {
+ names.reserve(read_schema_->num_fields());
+ // Native merge requires distinct fields; output alignment
restores repeated columns.
+ std::vector<uint8_t> selected_columns(file_schema_->num_fields(),
0);
+ for (const std::shared_ptr<arrow::Field>& field :
read_schema_->fields()) {
+ const int32_t field_index =
file_schema_->GetFieldIndex(field->name());
+ if (field_index >= 0 && selected_columns[field_index] == 0) {
+ names.push_back(field->name());
+ selected_columns[field_index] = 1;
+ }
+ }
+ if (!names.empty()) {
+ name_ptrs.reserve(names.size());
+ for (const std::string& name : names) {
+ name_ptrs.push_back(name.c_str());
+ }
+ root = VxExpressionPtr(vx_expression_root(),
vx_expression_free);
+ projection = VxExpressionPtr(
+ vx_expression_select(name_ptrs.data(), name_ptrs.size(),
root.get()),
+ vx_expression_free);
+ options.projection = projection.get();
+ }
+ }
+ // These FFI modes take individual indices. A dense bitmap sends only
excluded rows;
+ // an all-selected bitmap needs no indices and uses the default
INCLUDE_ALL mode.
+ if (has_selection_ && !selection_indices_.empty()) {
+ options.selection.idx = selection_indices_.data();
+ options.selection.idx_len = selection_indices_.size();
+ options.selection.include =
+ selection_excludes_ ? VX_SELECTION_EXCLUDE_RANGE :
VX_SELECTION_INCLUDE_RANGE;
+ }
+ // Filtering drops rows; include_row_idx preserves their original
physical positions.
+ // The scan clones the filter, so it can be freed after scan creation.
+ VxExpressionPtr filter(nullptr, vx_expression_free);
+ // The hidden index must not collide with a file field or a requested
missing field.
+ const bool has_row_idx_column =
+ file_schema_->GetFieldIndex(kRowIdxColumn) >= 0 ||
+ (read_schema_ != nullptr &&
!read_schema_->GetAllFieldIndices(kRowIdxColumn).empty());
+ if (!has_row_idx_column) {
Review Comment:
With schema validation in place, projection now uses read field names
directly instead of copying them.
##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
Review Comment:
Extracted scan creation into `CreateScan()`, leaving partition and stream
advancement in `OpenNextPartitionStream()`.
##########
src/paimon/format/vortex/vortex_file_batch_reader.cpp:
##########
@@ -448,24 +409,68 @@ Result<std::unique_ptr<::ArrowSchema>>
VortexFileBatchReader::GetFileSchema() co
Status VortexFileBatchReader::SetReadSchema(
::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
const std::optional<RoaringBitmap32>& selection_bitmap) {
+ if (closed_) {
+ return Status::Invalid("Vortex reader is closed");
+ }
if (read_schema == nullptr) {
return Status::Invalid("Vortex read schema is nullptr");
}
- // TODO(zhouyc-ali): push the row selection into the Vortex scan
- (void)selection_bitmap;
- // No pushdown: the whole file is scanned and reading restarts from row 0,
so drop the
- // single-use scan; the next read re-creates it.
+ std::vector<uint64_t> selection_indices;
+ bool selection_excludes = false;
+ uint64_t selected_row_count = total_rows_;
+ if (selection_bitmap) {
+ const int32_t cardinality = selection_bitmap->Cardinality();
+ if (cardinality < 0 || static_cast<uint64_t>(cardinality) >
total_rows_) {
+ return Status::Invalid("Vortex selection cardinality exceeds the
file row count");
+ }
+ selected_row_count = static_cast<uint64_t>(cardinality);
+ selection_excludes = selected_row_count > total_rows_ / 2;
+ selection_indices.reserve(static_cast<size_t>(
+ selection_excludes ? total_rows_ - selected_row_count :
selected_row_count));
+ uint64_t next_row = 0;
+ const auto end = selection_bitmap->End();
+ for (auto iter = selection_bitmap->Begin(); iter != end; ++iter) {
+ const auto row_id = static_cast<uint64_t>(*iter);
+ if (row_id >= total_rows_) {
+ return Status::Invalid(
+ fmt::format("Vortex selected row {} is out of range {}",
row_id, total_rows_));
+ }
+ if (selection_excludes) {
+ // Enumerate gaps directly instead of first expanding all
selected rows.
+ for (; next_row < row_id; ++next_row) {
+ selection_indices.push_back(next_row);
+ }
+ next_row = row_id + 1;
+ } else {
+ selection_indices.push_back(row_id);
+ }
+ }
+ if (selection_excludes) {
+ for (; next_row < total_rows_; ++next_row) {
+ selection_indices.push_back(next_row);
+ }
+ }
+ }
+ // Leave the active scan and its row mapping unchanged if validation or
import fails.
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema>
imported_read_schema,
arrow::ImportSchema(read_schema));
+ // Recreate the single-use scan on the next read with the new projection
and predicates.
read_schema_ = std::move(imported_read_schema);
predicate_filter_ = std::dynamic_pointer_cast<PredicateFilter>(predicate);
+ has_selection_ = selection_bitmap.has_value();
+ selection_excludes_ = selection_excludes;
+ selected_row_count_ = selected_row_count;
+ selection_indices_ = std::move(selection_indices);
+ metrics_->SetCounter("vortex.selection-index-count",
selection_indices_.size());
ReleaseStream();
Review Comment:
Centralized metric names in `vortex_defs.h` and renamed the counter to
`vortex.selection-row-count`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]