SteNicholas commented on code in PR #564:
URL: https://github.com/apache/paimon-cpp/pull/564#discussion_r4596345353
##########
src/paimon/common/reader/concat_batch_reader.cpp:
##########
@@ -33,78 +35,154 @@ class MemoryPool;
ConcatBatchReader::ConcatBatchReader(std::vector<std::unique_ptr<BatchReader>>&&
readers,
const std::shared_ptr<arrow::MemoryPool>&
arrow_pool)
- : arrow_pool_(arrow_pool),
- finished_reader_metrics_(std::make_shared<MetricsImpl>()),
- readers_(std::move(readers)),
- file_readers_(readers_.size(), nullptr),
- current_(0) {
- // Non-file readers keep their null slots so the warmup window stays
aligned with readers_.
- for (size_t i = 0; i < readers_.size(); i++) {
- if (auto* file_reader =
dynamic_cast<FileBatchReader*>(readers_[i].get())) {
- file_readers_[i] = file_reader;
+ : arrow_pool_(arrow_pool),
finished_reader_metrics_(std::make_shared<MetricsImpl>()) {
+ for (auto& reader : readers) {
+ readers_.emplace_back();
+ readers_.back().file_reader =
dynamic_cast<FileBatchReader*>(reader.get());
+ readers_.back().reader = std::move(reader);
+ }
+}
+
+ConcatBatchReader::~ConcatBatchReader() {
+ ConcatBatchReader::Close();
+}
+
+Result<std::unique_ptr<ConcatBatchReader>> ConcatBatchReader::CreateLazy(
+ size_t reader_count, ReaderFactory factory,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool,
+ const std::shared_ptr<Executor>& build_executor) {
+ if (!factory || !build_executor) {
+ return Status::Invalid("lazy reader requires a factory and a build
executor");
+ }
+ auto reader =
std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>(),
+ arrow_pool);
+ reader->preparation_ = std::make_shared<Preparation>(std::move(factory));
+ reader->build_executor_ = build_executor;
+ reader->reader_count_ = reader_count;
+ reader->FillWindow();
+ for (auto& state : reader->readers_) {
+ PAIMON_RETURN_NOT_OK(reader->ResolveReader(&state, /*wait=*/true));
+ }
+ return reader;
+}
+
+void ConcatBatchReader::FillWindow() {
+ while (next_reader_ < reader_count_ && readers_.size() <
kPreparationWindow) {
+ readers_.emplace_back();
+ readers_.back().future =
+ Via(build_executor_.get(),
+ [preparation = preparation_,
+ index = next_reader_++]() mutable ->
Result<std::unique_ptr<BatchReader>> {
+ // Release the task's creator before Via publishes its
result, so draining
+ // the future also releases the background owner of the
creation context.
+ auto retained = std::move(preparation);
+ if (retained->cancelled.load()) {
+ return std::unique_ptr<BatchReader>();
+ }
+ return retained->factory(index);
+ });
+ }
+}
+
+Status ConcatBatchReader::ResolveReader(ReaderState* state, bool wait) {
+ if (state->future.valid() &&
+ (wait || state->future.wait_for(std::chrono::seconds(0)) ==
std::future_status::ready)) {
+ auto result = state->future.get();
+ if (!result.ok()) {
+ state->failure = result.status();
+ } else {
+ state->reader = std::move(result).value();
+ state->file_reader =
dynamic_cast<FileBatchReader*>(state->reader.get());
}
}
+ return state->failure;
}
Result<BatchReader::ReadBatch> ConcatBatchReader::NextBatch() {
PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
NextBatchWithBitmap());
- PAIMON_ASSIGN_OR_RAISE(
- BatchReader::ReadBatch batch,
- ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap),
arrow_pool_));
- return batch;
+ auto result =
ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), arrow_pool_);
+ if (!result.ok()) {
+ failure_ = result.status();
+ Close();
+ }
+ return result;
}
void ConcatBatchReader::Close() {
- for (; current_ < readers_.size(); current_++) {
- CloseAndReleaseReader(current_);
+ if (preparation_) {
+ preparation_->cancelled.store(true);
}
+ for (auto& state : readers_) {
+ (void)ResolveReader(&state, /*wait=*/true);
Review Comment:
Fixed in
[31535f80](https://github.com/apache/paimon-cpp/pull/564/commits/31535f80dc44f4d3c3454365a096483092e94356).
`Close()` now catches construction exceptions per future and continues
draining the remaining tasks and releasing successfully prepared readers.
Added `LazyEarlyCloseAndDestructionDrainExceptionalBuilds` to inject a
deferred `std::bad_alloc` and verify early destruction, explicit/repeated
close, closure of readers built after the failure, and creator/resource
release. The regression failed before the fix and now passes. All 12
`ConcatBatchReaderTest` and 20 `RawFileSplitReadTest` cases pass in focused
builds; changed-file pre-commit checks and `git diff --check` also pass.
Full common-suite validation remains blocked on this macOS setup by existing
`INFINITY`/`NAN` compilation errors and a `Literal<long>` link error.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]