zjw1111 commented on code in PR #370:
URL: https://github.com/apache/paimon-cpp/pull/370#discussion_r4056269117
##########
src/paimon/core/operation/abstract_split_read.cpp:
##########
@@ -138,13 +168,92 @@ Result<std::vector<std::unique_ptr<FileBatchReader>>>
AbstractSplitRead::CreateR
}
std::unique_ptr<FileBatchReader> file_reader =
std::move(built).value();
if (file_reader) {
- raw_file_readers.push_back(std::move(file_reader));
+ raw_file_readers.push_back({file, std::move(file_reader)});
}
}
PAIMON_RETURN_NOT_OK(first_error);
return std::move(raw_file_readers);
}
+Result<std::unique_ptr<FileBatchReader>>
AbstractSplitRead::ApplyIndexAndDvReaderIfNeeded(
+ std::unique_ptr<FileBatchReader>&& file_reader, const
std::shared_ptr<DataFileMeta>& file,
+ const std::shared_ptr<arrow::Schema>& data_schema,
+ const std::shared_ptr<arrow::Schema>& read_schema, const
std::shared_ptr<Predicate>& predicate,
+ DeletionVector::Factory dv_factory, const
std::optional<std::vector<Range>>& row_ranges,
+ const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
+ std::shared_ptr<FileIndexResult> file_index_result;
+ if (options_.FileIndexReadEnabled()) {
+ PAIMON_ASSIGN_OR_RAISE(
+ file_index_result,
+ FileIndexEvaluator::Evaluate(data_schema, predicate,
data_file_path_factory, file,
+ options_.GetFileSystem(), pool_));
+ PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain());
+ if (!is_remain) {
+ return std::unique_ptr<FileBatchReader>();
+ }
+ }
+
+ const RoaringBitmap32* selection = nullptr;
+ if (auto* bitmap_file_index =
dynamic_cast<BitmapIndexResult*>(file_index_result.get())) {
+ PAIMON_ASSIGN_OR_RAISE(selection, bitmap_file_index->GetBitmap());
+ }
+
+ std::optional<RoaringBitmap32> row_ranges_selection;
+ if (row_ranges) {
+ RoaringBitmap32 row_ranges_bitmap;
+ for (const Range& range : row_ranges.value()) {
+ row_ranges_bitmap.AddRange(static_cast<int32_t>(range.from),
+ static_cast<int32_t>(range.to + 1));
+ }
+ row_ranges_selection = selection ? RoaringBitmap32::And(*selection,
row_ranges_bitmap)
+ : std::move(row_ranges_bitmap);
+ selection = &row_ranges_selection.value();
+ }
+
+ std::shared_ptr<DeletionVector> deletion_vector;
+ if (dv_factory) {
+ PAIMON_ASSIGN_OR_RAISE(deletion_vector, dv_factory(file->file_name));
+ }
+ const RoaringBitmap32* deletion = nullptr;
+ if (auto* bitmap_dv =
dynamic_cast<BitmapDeletionVector*>(deletion_vector.get())) {
+ deletion = bitmap_dv->GetBitmap();
+ }
+
+ std::optional<RoaringBitmap32> actual_selection;
+ if (selection && deletion) {
+ actual_selection = RoaringBitmap32::AndNot(*selection, *deletion);
+ } else if (selection) {
+ actual_selection = *selection;
+ } else if (deletion) {
+ actual_selection = *deletion;
+ PAIMON_ASSIGN_OR_RAISE(uint64_t num_rows,
file_reader->GetNumberOfRows());
+ actual_selection->Flip(0, num_rows);
+ }
+
+ if (actual_selection && actual_selection->IsEmpty()) {
+ return std::unique_ptr<FileBatchReader>();
+ }
+
+ ::ArrowSchema c_read_schema;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema,
&c_read_schema));
+ PAIMON_RETURN_NOT_OK(file_reader->SetReadSchema(&c_read_schema, predicate,
actual_selection));
+
+ std::unique_ptr<FileBatchReader> reader;
+ if (!file_reader->SupportPreciseBitmapSelection() && actual_selection) {
+ reader =
std::make_unique<ApplyBitmapIndexBatchReader>(std::move(file_reader),
Review Comment:
This changes the DV-only merge path from `ApplyDeletionVectorBatchReader` to
`ApplyBitmapIndexBatchReader`. The latter starts at `bitmap_.Begin()` for every
batch and advances through all selected row IDs preceding that batch. With a
sparse DV, its complement contains almost every row, so reading N rows in
batches of B requires O(N² / B) bitmap iterator increments, instead of the
previous per-row DV membership checks. Both Parquet and ORC report
`SupportPreciseBitmapSelection() == false`, so this regression also affects
reads with file indexes disabled or no predicate.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
