SteNicholas commented on code in PR #564:
URL: https://github.com/apache/paimon-cpp/pull/564#discussion_r4596345353


##########
src/paimon/common/reader/concat_batch_reader.cpp:
##########
@@ -33,78 +35,154 @@ class MemoryPool;
 
 
ConcatBatchReader::ConcatBatchReader(std::vector<std::unique_ptr<BatchReader>>&&
 readers,
                                      const std::shared_ptr<arrow::MemoryPool>& 
arrow_pool)
-    : arrow_pool_(arrow_pool),
-      finished_reader_metrics_(std::make_shared<MetricsImpl>()),
-      readers_(std::move(readers)),
-      file_readers_(readers_.size(), nullptr),
-      current_(0) {
-    // Non-file readers keep their null slots so the warmup window stays 
aligned with readers_.
-    for (size_t i = 0; i < readers_.size(); i++) {
-        if (auto* file_reader = 
dynamic_cast<FileBatchReader*>(readers_[i].get())) {
-            file_readers_[i] = file_reader;
+    : arrow_pool_(arrow_pool), 
finished_reader_metrics_(std::make_shared<MetricsImpl>()) {
+    for (auto& reader : readers) {
+        readers_.emplace_back();
+        readers_.back().file_reader = 
dynamic_cast<FileBatchReader*>(reader.get());
+        readers_.back().reader = std::move(reader);
+    }
+}
+
+ConcatBatchReader::~ConcatBatchReader() {
+    ConcatBatchReader::Close();
+}
+
+Result<std::unique_ptr<ConcatBatchReader>> ConcatBatchReader::CreateLazy(
+    size_t reader_count, ReaderFactory factory,
+    const std::shared_ptr<arrow::MemoryPool>& arrow_pool,
+    const std::shared_ptr<Executor>& build_executor) {
+    if (!factory || !build_executor) {
+        return Status::Invalid("lazy reader requires a factory and a build 
executor");
+    }
+    auto reader = 
std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>(),
+                                                      arrow_pool);
+    reader->preparation_ = std::make_shared<Preparation>(std::move(factory));
+    reader->build_executor_ = build_executor;
+    reader->reader_count_ = reader_count;
+    reader->FillWindow();
+    for (auto& state : reader->readers_) {
+        PAIMON_RETURN_NOT_OK(reader->ResolveReader(&state, /*wait=*/true));
+    }
+    return reader;
+}
+
+void ConcatBatchReader::FillWindow() {
+    while (next_reader_ < reader_count_ && readers_.size() < 
kPreparationWindow) {
+        readers_.emplace_back();
+        readers_.back().future =
+            Via(build_executor_.get(),
+                [preparation = preparation_,
+                 index = next_reader_++]() mutable -> 
Result<std::unique_ptr<BatchReader>> {
+                    // Release the task's creator before Via publishes its 
result, so draining
+                    // the future also releases the background owner of the 
creation context.
+                    auto retained = std::move(preparation);
+                    if (retained->cancelled.load()) {
+                        return std::unique_ptr<BatchReader>();
+                    }
+                    return retained->factory(index);
+                });
+    }
+}
+
+Status ConcatBatchReader::ResolveReader(ReaderState* state, bool wait) {
+    if (state->future.valid() &&
+        (wait || state->future.wait_for(std::chrono::seconds(0)) == 
std::future_status::ready)) {
+        auto result = state->future.get();
+        if (!result.ok()) {
+            state->failure = result.status();
+        } else {
+            state->reader = std::move(result).value();
+            state->file_reader = 
dynamic_cast<FileBatchReader*>(state->reader.get());
         }
     }
+    return state->failure;
 }
 
 Result<BatchReader::ReadBatch> ConcatBatchReader::NextBatch() {
     PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
                            NextBatchWithBitmap());
-    PAIMON_ASSIGN_OR_RAISE(
-        BatchReader::ReadBatch batch,
-        ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), 
arrow_pool_));
-    return batch;
+    auto result = 
ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), arrow_pool_);
+    if (!result.ok()) {
+        failure_ = result.status();
+        Close();
+    }
+    return result;
 }
 
 void ConcatBatchReader::Close() {
-    for (; current_ < readers_.size(); current_++) {
-        CloseAndReleaseReader(current_);
+    if (preparation_) {
+        preparation_->cancelled.store(true);
     }
+    for (auto& state : readers_) {
+        (void)ResolveReader(&state, /*wait=*/true);

Review Comment:
   Fixed in 
[31535f80](https://github.com/apache/paimon-cpp/pull/564/commits/31535f80dc44f4d3c3454365a096483092e94356).
 `Close()` now catches construction exceptions per future and continues 
draining the remaining tasks and releasing successfully prepared readers.
   
   Added `LazyEarlyCloseAndDestructionDrainExceptionalBuilds` to inject a 
deferred `std::bad_alloc` and verify early destruction, explicit/repeated 
close, closure of readers built after the failure, and creator/resource 
release. The regression failed before the fix and now passes. All 12 
`ConcatBatchReaderTest` and 20 `RawFileSplitReadTest` cases pass in focused 
builds; changed-file pre-commit checks and `git diff --check` also pass.
   
   Full common-suite validation remains blocked on this macOS setup by existing 
`INFINITY`/`NAN` compilation errors and a `Literal<long>` link error.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to