zjw1111 commented on code in PR #504:
URL: https://github.com/apache/paimon-cpp/pull/504#discussion_r4510169160
##########
src/paimon/core/realtime/arrow_realtime_store.cpp:
##########
@@ -525,41 +532,262 @@ class ArrowRealtimeStore::AppendQueryBatchReader :
public BatchReader {
void Close() override {
view_ = nullptr;
spill_reader_.reset();
+ candidates_.reset();
}
private:
+ Result<bool> PrepareSegment(const std::shared_ptr<Segment>& segment) {
+ const OffsetRange segment_range = segment->GetOffsetRange();
+ if (segment_range.end <= offset_range_.begin || segment_range.begin >=
offset_range_.end) {
+ return false;
+ }
+ if (!segment_initialized_) {
+ if (segment->GetIndex()) {
+ PAIMON_ASSIGN_OR_RAISE(
+ candidates_, RealtimeIndexEvaluator::Evaluate(
+ segment->GetIndex(), predicate_,
+ OffsetRange(std::max(segment_range.begin,
offset_range_.begin),
+ std::min(segment_range.end,
offset_range_.end))));
+ }
+ segment_initialized_ = true;
+ }
+ return !candidates_ || !candidates_->IsEmpty();
+ }
+
+ Result<bool> SeekNextBatch(const std::shared_ptr<Segment>& segment) {
+ const std::shared_ptr<MemorySegment> memory_segment =
+ std::dynamic_pointer_cast<MemorySegment>(segment);
+ if (memory_segment) {
+ return SeekNextBatch(memory_segment->GetBatches());
+ }
+ const std::shared_ptr<SpilledSegment> spilled_segment =
+ std::dynamic_pointer_cast<SpilledSegment>(segment);
+ if (spilled_segment) {
+ return SeekNextBatch(spilled_segment->GetBatches());
+ }
+ return Status::Invalid("unknown Arrow real-time segment type");
+ }
+
+ template <typename Batch>
+ bool SeekNextBatch(const std::vector<Batch>& batches) {
+ if (next_batch_ >= batches.size()) {
+ return false;
+ }
+ const OffsetRange current_range = batches[next_batch_].offset_range;
+ int64_t first_offset = std::max(current_range.begin,
offset_range_.begin);
+ if (candidates_) {
+ const RoaringBitmap64::Iterator first =
candidates_->EqualOrLarger(first_offset);
+ if (first == candidates_->End() || *first >= offset_range_.end) {
+ return false;
+ }
+ first_offset = *first;
+ }
+ const auto next = std::lower_bound(
+ batches.begin() + next_batch_, batches.end(), first_offset,
+ [](const Batch& batch, int64_t offset) { return
batch.offset_range.end <= offset; });
+ next_batch_ = next - batches.begin();
+ return next_batch_ < batches.size();
+ }
+
+ Result<std::optional<ReadBatchWithBitmap>> TryReadBatch(const
std::shared_ptr<Segment>& segment,
+ size_t
batch_index) {
+ const std::shared_ptr<MemorySegment> memory_segment =
+ std::dynamic_pointer_cast<MemorySegment>(segment);
+ const std::shared_ptr<SpilledSegment> spilled_segment =
+ std::dynamic_pointer_cast<SpilledSegment>(segment);
+ int64_t row_count;
+ OffsetRange batch_range(0, 0);
+ const std::optional<BatchStatistics>* statistics;
+ std::shared_ptr<arrow::StructArray> source;
+ if (memory_segment) {
+ const StoredBatch& stored =
memory_segment->GetBatches()[batch_index];
+ row_count = stored.data->length();
+ statistics = &stored.statistics;
+ source = stored.data;
+ batch_range = stored.offset_range;
+ } else if (spilled_segment) {
+ const SpilledBatch& stored =
spilled_segment->GetBatches()[batch_index];
+ row_count = stored.row_count;
+ statistics = &stored.statistics;
+ batch_range = stored.offset_range;
+ } else {
+ return Status::Invalid("unknown Arrow real-time segment type");
+ }
+ const OffsetRange visible_range(std::max(batch_range.begin,
offset_range_.begin),
+ std::min(batch_range.end,
offset_range_.end));
+ if (visible_range.begin >= visible_range.end ||
+ (candidates_ && !candidates_->ContainsAny(visible_range.begin,
visible_range.end))) {
+ return std::optional<ReadBatchWithBitmap>();
+ }
+ PAIMON_ASSIGN_OR_RAISE(bool may_match,
+ ArrowRealtimeStore::MayMatchStatistics(
+ row_count, *statistics, read_schema_,
predicate_filter_,
+ statistics_mapping_, memory_pool_));
+ if (!may_match) {
+ return std::optional<ReadBatchWithBitmap>();
+ }
+ if (spilled_segment) {
+ PAIMON_ASSIGN_OR_RAISE(source, ReadSpilledBatch(spilled_segment,
batch_index));
+ }
+ RoaringBitmap32 selection = SelectRows(source, visible_range);
+ if (selection.IsEmpty()) {
+ return std::optional<ReadBatchWithBitmap>();
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::StructArray> output,
+ ProjectBatch(source, read_schema_,
arrow_pool_.get()));
+ PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, ExportStructBatch(output,
arrow_pool_));
+ return std::optional<ReadBatchWithBitmap>(
+ ReadBatchWithBitmap(std::move(batch), std::move(selection)));
+ }
+
+ Result<std::shared_ptr<arrow::StructArray>> ReadSpilledBatch(
+ const std::shared_ptr<SpilledSegment>& segment, size_t batch_index) {
+ if (!spill_reader_) {
+ PAIMON_ASSIGN_OR_RAISE(spill_reader_,
+
ArrowIpcFileReader::Open(segment->GetFile()->GetFileSystem(),
+
segment->GetFile()->GetPath(),
+
/*use_threads=*/false, arrow_pool_));
+ if (spill_reader_->GetRecordBatchCount() !=
+ static_cast<int32_t>(segment->GetBatches().size())) {
+ return Status::Invalid(
+ "real-time spill file batch count does not match segment
metadata");
+ }
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::RecordBatch>
record_batch,
+ spill_reader_->ReadRecordBatch(batch_index));
+ return ToStructArray(record_batch);
+ }
+
+ void NextSegment() {
+ ++next_segment_;
+ next_batch_ = 0;
+ spill_reader_.reset();
+ candidates_.reset();
+ segment_initialized_ = false;
+ }
+
+ RoaringBitmap32 SelectRows(const std::shared_ptr<arrow::StructArray>&
source,
+ const OffsetRange& visible_range) const {
+ // TODO(xinyu.lxy): Push down the real-time DV to avoid another
offset-to-row mapping
+ // upstream.
+ RoaringBitmap32 selection;
+ if (source->length() == 0) {
+ return selection;
+ }
+ const std::shared_ptr<arrow::Int64Array> offsets =
checked_pointer_cast<arrow::Int64Array>(
+ source->GetFieldByName(SpecialFields::RealtimeOffset().Name()));
+ const int64_t* values = offsets->raw_values();
+ const int64_t* begin =
+ std::lower_bound(values, values + offsets->length(),
visible_range.begin);
+ const int64_t* end = std::lower_bound(begin, values +
offsets->length(), visible_range.end);
+ if (!candidates_) {
+ selection.AddRange(begin - values, end - values);
+ return selection;
+ }
+ // Offsets may have gaps: sparse candidates seek by offset; dense
candidates scan.
+ const RoaringBitmap64::Iterator sentinel = candidates_->End();
+ const RoaringBitmap64::Iterator first =
candidates_->EqualOrLarger(visible_range.begin);
+ RoaringBitmap64::Iterator probe = first;
+ const int64_t dense_threshold = std::max<int64_t>(1, (end - begin) /
4);
+ int64_t candidate_count = 0;
+ while (candidate_count < dense_threshold && probe != sentinel &&
+ *probe < visible_range.end) {
+ ++candidate_count;
+ ++probe;
+ }
+ if (candidate_count < dense_threshold) {
+ for (RoaringBitmap64::Iterator iter = first;
+ iter != sentinel && *iter < visible_range.end; ++iter) {
+ begin = std::lower_bound(begin, end, *iter);
+ if (begin != end && *begin == *iter) {
+ selection.Add(begin - values);
+ }
+ }
+ } else {
+ for (const int64_t* row = begin; row != end; ++row) {
+ if (candidates_->Contains(*row)) {
+ selection.Add(row - values);
+ }
+ }
+ }
+ return selection;
+ }
+
std::shared_ptr<ReadView> view_;
std::shared_ptr<arrow::Schema> read_schema_;
std::shared_ptr<arrow::MemoryPool> arrow_pool_;
std::shared_ptr<MemoryPool> memory_pool_;
std::shared_ptr<PredicateFilter> predicate_filter_;
std::vector<int32_t> statistics_mapping_;
+ std::shared_ptr<Predicate> predicate_;
+ OffsetRange offset_range_;
+ std::optional<RoaringBitmap64> candidates_;
+ bool segment_initialized_ = false;
std::shared_ptr<Metrics> metrics_;
std::unique_ptr<ArrowIpcFileReader> spill_reader_;
size_t next_segment_ = 0;
size_t next_batch_ = 0;
};
+Result<std::shared_ptr<ArrowRealtimeStore>> ArrowRealtimeStore::Create(
+ const std::shared_ptr<arrow::Schema>& write_schema, RealtimeStoreMode mode,
+ StatisticsMode statistics_mode, const std::string& temp_directory,
+ const std::shared_ptr<FileSystem>& spill_file_system, const std::string&
spill_compression,
+ int32_t spill_compression_level, const std::map<std::string, std::string>&
index_options,
+ const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+ if (!write_schema || !memory_pool || !arrow_pool) {
Review Comment:
`ArrowRealtimeStoreFactory::Create()` already validates the schema and
memory pool and constructs `arrow_pool` with `GetArrowPool()`. It also requires
a file system when spill is enabled. These checks repeat guarantees established
by that caller.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]