lucasfang commented on code in PR #564:
URL: https://github.com/apache/paimon-cpp/pull/564#discussion_r4595562424
##########
src/paimon/common/reader/concat_batch_reader.cpp:
##########
@@ -33,78 +35,154 @@ class MemoryPool;
ConcatBatchReader::ConcatBatchReader(std::vector<std::unique_ptr<BatchReader>>&&
readers,
const std::shared_ptr<arrow::MemoryPool>&
arrow_pool)
- : arrow_pool_(arrow_pool),
- finished_reader_metrics_(std::make_shared<MetricsImpl>()),
- readers_(std::move(readers)),
- file_readers_(readers_.size(), nullptr),
- current_(0) {
- // Non-file readers keep their null slots so the warmup window stays
aligned with readers_.
- for (size_t i = 0; i < readers_.size(); i++) {
- if (auto* file_reader =
dynamic_cast<FileBatchReader*>(readers_[i].get())) {
- file_readers_[i] = file_reader;
+ : arrow_pool_(arrow_pool),
finished_reader_metrics_(std::make_shared<MetricsImpl>()) {
+ for (auto& reader : readers) {
+ readers_.emplace_back();
+ readers_.back().file_reader =
dynamic_cast<FileBatchReader*>(reader.get());
+ readers_.back().reader = std::move(reader);
+ }
+}
+
+ConcatBatchReader::~ConcatBatchReader() {
+ ConcatBatchReader::Close();
+}
+
+Result<std::unique_ptr<ConcatBatchReader>> ConcatBatchReader::CreateLazy(
+ size_t reader_count, ReaderFactory factory,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool,
+ const std::shared_ptr<Executor>& build_executor) {
+ if (!factory || !build_executor) {
+ return Status::Invalid("lazy reader requires a factory and a build
executor");
+ }
+ auto reader =
std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>(),
+ arrow_pool);
+ reader->preparation_ = std::make_shared<Preparation>(std::move(factory));
+ reader->build_executor_ = build_executor;
+ reader->reader_count_ = reader_count;
+ reader->FillWindow();
+ for (auto& state : reader->readers_) {
+ PAIMON_RETURN_NOT_OK(reader->ResolveReader(&state, /*wait=*/true));
+ }
+ return reader;
+}
+
+void ConcatBatchReader::FillWindow() {
+ while (next_reader_ < reader_count_ && readers_.size() <
kPreparationWindow) {
+ readers_.emplace_back();
+ readers_.back().future =
+ Via(build_executor_.get(),
+ [preparation = preparation_,
+ index = next_reader_++]() mutable ->
Result<std::unique_ptr<BatchReader>> {
+ // Release the task's creator before Via publishes its
result, so draining
+ // the future also releases the background owner of the
creation context.
+ auto retained = std::move(preparation);
+ if (retained->cancelled.load()) {
+ return std::unique_ptr<BatchReader>();
+ }
+ return retained->factory(index);
+ });
+ }
+}
+
+Status ConcatBatchReader::ResolveReader(ReaderState* state, bool wait) {
+ if (state->future.valid() &&
+ (wait || state->future.wait_for(std::chrono::seconds(0)) ==
std::future_status::ready)) {
+ auto result = state->future.get();
+ if (!result.ok()) {
+ state->failure = result.status();
+ } else {
+ state->reader = std::move(result).value();
+ state->file_reader =
dynamic_cast<FileBatchReader*>(state->reader.get());
}
}
+ return state->failure;
}
Result<BatchReader::ReadBatch> ConcatBatchReader::NextBatch() {
PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
NextBatchWithBitmap());
- PAIMON_ASSIGN_OR_RAISE(
- BatchReader::ReadBatch batch,
- ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap),
arrow_pool_));
- return batch;
+ auto result =
ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), arrow_pool_);
+ if (!result.ok()) {
+ failure_ = result.status();
+ Close();
+ }
+ return result;
}
void ConcatBatchReader::Close() {
- for (; current_ < readers_.size(); current_++) {
- CloseAndReleaseReader(current_);
+ if (preparation_) {
+ preparation_->cancelled.store(true);
}
+ for (auto& state : readers_) {
+ (void)ResolveReader(&state, /*wait=*/true);
Review Comment:
**[P2] Handle construction exceptions during destruction**
If a deferred reader construction throws—for example, `std::bad_alloc` while
allocating a deletion-vector buffer—and the caller destroys the reader before
consuming that future, `~ConcatBatchReader()` calls `Close()`, where
`future.get()` rethrows. The exception escapes the implicitly `noexcept`
destructor and triggers `std::terminate`, turning a recoverable construction
failure into process termination.
Please handle exceptional futures during cleanup and continue draining the
remaining tasks, ensuring destruction cannot throw. A regression test covering
deferred allocation failure followed by early destruction would help verify
this path.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]