SteNicholas commented on code in PR #513:
URL: https://github.com/apache/paimon-cpp/pull/513#discussion_r4476202443


##########
src/paimon/common/reader/warmup_file_batch_reader.cpp:
##########
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+
+namespace paimon {
+
+WarmupFileBatchReader::WarmupFileBatchReader(std::shared_ptr<PrefetchFileBatchReader>
 reader,
+                                             std::shared_ptr<FileReadAhead> 
read_ahead,
+                                             WarmupLevel level,
+                                             
std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      read_ahead_(std::move(read_ahead)),
+      level_(level),
+      arrow_pool_(std::move(arrow_pool)) {}
+
+WarmupFileBatchReader::~WarmupFileBatchReader() {
+    ResetWarmup();
+    reader_->SetPreBufferRangeCallback(nullptr);
+    read_ahead_->ReleaseBuffers();
+}
+
+void WarmupFileBatchReader::Join() const {
+    if (thread_) {
+        thread_->join();
+        thread_.reset();
+    }
+}
+
+void WarmupFileBatchReader::ResetWarmup() {
+    Join();
+    if (batch_ && batch_->ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch_.value()).value().first);
+    }
+    batch_.reset();
+    prepared_ = false;
+    started_ = false;
+    batch_returned_ = false;
+    read_status_ = Status::OK();
+}
+
+void WarmupFileBatchReader::Prepare() {
+    prepared_ = true;
+}
+
+void WarmupFileBatchReader::Warmup() {
+    if (!prepared_ || closed_ || started_ || level_ == WarmupLevel::NONE) {
+        return;
+    }
+    if (level_ == WarmupLevel::RAW) {
+        (void)read_ahead_->Warmup(reader_.get());
+    } else {
+        started_ = true;
+        thread_ = std::make_unique<std::thread>([this]() { 
batch_.emplace(ReadOneBatch()); });

Review Comment:
   A `MemoryPool` that throws `std::bad_alloc` on a cache allocation makes RAW 
`Warmup()` throw on the caller thread. In DECODED mode I reproduced 
`std::terminate` when the same exception escaped this thread entry point. That 
can terminate a query while it is warming a file it will never consume. Please 
handle allocation and thread-start failures at the warmup boundary, preserve a 
`Status` for the first read that consumes the file, and cover closing an 
unconsumed reader after failure.



##########
src/paimon/common/reader/warmup_file_batch_reader.cpp:
##########
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+
+namespace paimon {
+
+WarmupFileBatchReader::WarmupFileBatchReader(std::shared_ptr<PrefetchFileBatchReader>
 reader,
+                                             std::shared_ptr<FileReadAhead> 
read_ahead,
+                                             WarmupLevel level,
+                                             
std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      read_ahead_(std::move(read_ahead)),
+      level_(level),
+      arrow_pool_(std::move(arrow_pool)) {}
+
+WarmupFileBatchReader::~WarmupFileBatchReader() {
+    ResetWarmup();
+    reader_->SetPreBufferRangeCallback(nullptr);
+    read_ahead_->ReleaseBuffers();
+}
+
+void WarmupFileBatchReader::Join() const {
+    if (thread_) {
+        thread_->join();
+        thread_.reset();
+    }
+}
+
+void WarmupFileBatchReader::ResetWarmup() {
+    Join();
+    if (batch_ && batch_->ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch_.value()).value().first);
+    }
+    batch_.reset();
+    prepared_ = false;
+    started_ = false;
+    batch_returned_ = false;
+    read_status_ = Status::OK();
+}
+
+void WarmupFileBatchReader::Prepare() {
+    prepared_ = true;
+}
+
+void WarmupFileBatchReader::Warmup() {
+    if (!prepared_ || closed_ || started_ || level_ == WarmupLevel::NONE) {
+        return;
+    }
+    if (level_ == WarmupLevel::RAW) {
+        (void)read_ahead_->Warmup(reader_.get());
+    } else {
+        started_ = true;
+        thread_ = std::make_unique<std::thread>([this]() { 
batch_.emplace(ReadOneBatch()); });
+    }
+}
+
+Result<BatchReader::ReadBatchWithBitmap> WarmupFileBatchReader::ReadOneBatch() 
{
+    PAIMON_RETURN_NOT_OK(read_ahead_->Warmup(reader_.get()));
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, 
reader_->NextBatchWithBitmap());
+    Status status = read_ahead_->GetStatus();
+    if (!status.ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch.first));
+        return status;
+    }
+    return batch;
+}
+
+Result<BatchReader::ReadBatchWithBitmap> 
WarmupFileBatchReader::NextBatchWithBitmap() {
+    if (closed_) {
+        return Status::Invalid("cannot read a closed warmup reader");
+    }
+    PAIMON_RETURN_NOT_OK(read_status_);
+    // A normal read is synchronous unless Warmup() explicitly prepared the 
first batch.
+    started_ = true;
+    Join();
+    batch_returned_ = false;
+    Result<ReadBatchWithBitmap> result = batch_ ? std::move(batch_.value()) : 
ReadOneBatch();
+    batch_.reset();
+    if (result.ok() && !BatchReader::IsEofBatch(result.value())) {
+        batch_returned_ = true;
+    } else if (!result.ok()) {
+        read_status_ = result.status();
+    }
+    return result;
+}
+
+Result<BatchReader::ReadBatch> WarmupFileBatchReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, NextBatchWithBitmap());
+    return ReaderUtils::ApplyBitmapToReadBatch(std::move(batch), arrow_pool_);
+}
+
+Status WarmupFileBatchReader::SetReadSchema(::ArrowSchema* schema,
+                                            const std::shared_ptr<Predicate>& 
predicate,
+                                            const 
std::optional<RoaringBitmap32>& bitmap) {
+    ResetWarmup();
+    if (closed_) {
+        return Status::Invalid("cannot configure a closed warmup reader");
+    }
+    read_ahead_->Reset();
+    read_status_ = reader_->SetReadSchema(schema, predicate, bitmap);
+    PAIMON_RETURN_NOT_OK(read_status_);
+    Prepare();
+    return Status::OK();
+}
+
+std::shared_ptr<Metrics> WarmupFileBatchReader::GetReaderMetrics() const {
+    // The underlying reader may be decoding. Do not inspect its mutable 
metrics until it stops.
+    // Joining is reserved for operations needing the format reader, so a live 
snapshot contains
+    // only the thread-safe cache and I/O counters while the first batch is 
being prepared.
+    auto metrics = std::make_shared<MetricsImpl>();
+    if (!thread_) {

Review Comment:
   `thread_` stays set after decoding finishes until a later call joins it. I 
verified that a format counter visible before `Warmup()` becomes unavailable 
even after the first decode completes, then reappears when the consumer takes 
the batch. An aggregate can consequently drop while progress is being reported. 
Please retain the last safe format-metrics snapshot while decoding and refresh 
it once the task has completed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to