SteNicholas commented on code in PR #513:
URL: https://github.com/apache/paimon-cpp/pull/513#discussion_r4476202443


##########
src/paimon/common/reader/warmup_file_batch_reader.cpp:
##########
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+
+namespace paimon {
+
+WarmupFileBatchReader::WarmupFileBatchReader(std::shared_ptr<PrefetchFileBatchReader>
 reader,
+                                             std::shared_ptr<FileReadAhead> 
read_ahead,
+                                             WarmupLevel level,
+                                             
std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      read_ahead_(std::move(read_ahead)),
+      level_(level),
+      arrow_pool_(std::move(arrow_pool)) {}
+
+WarmupFileBatchReader::~WarmupFileBatchReader() {
+    ResetWarmup();
+    reader_->SetPreBufferRangeCallback(nullptr);
+    read_ahead_->ReleaseBuffers();
+}
+
+void WarmupFileBatchReader::Join() const {
+    if (thread_) {
+        thread_->join();
+        thread_.reset();
+    }
+}
+
+void WarmupFileBatchReader::ResetWarmup() {
+    Join();
+    if (batch_ && batch_->ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch_.value()).value().first);
+    }
+    batch_.reset();
+    prepared_ = false;
+    started_ = false;
+    batch_returned_ = false;
+    read_status_ = Status::OK();
+}
+
+void WarmupFileBatchReader::Prepare() {
+    prepared_ = true;
+}
+
+void WarmupFileBatchReader::Warmup() {
+    if (!prepared_ || closed_ || started_ || level_ == WarmupLevel::NONE) {
+        return;
+    }
+    if (level_ == WarmupLevel::RAW) {
+        (void)read_ahead_->Warmup(reader_.get());
+    } else {
+        started_ = true;
+        thread_ = std::make_unique<std::thread>([this]() { 
batch_.emplace(ReadOneBatch()); });

Review Comment:
   A `MemoryPool` that throws `std::bad_alloc` on a cache allocation makes RAW 
`Warmup()` throw on the caller thread. In DECODED mode I reproduced 
`std::terminate` when the same exception escaped this thread entry point. That 
can terminate a query while it is warming a file it will never consume. Please 
handle allocation and thread-start failures at the warmup boundary, preserve a 
`Status` for the first read that consumes the file, and cover closing an 
unconsumed reader after failure.



##########
src/paimon/common/reader/warmup_file_batch_reader.cpp:
##########
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+
+namespace paimon {
+
+WarmupFileBatchReader::WarmupFileBatchReader(std::shared_ptr<PrefetchFileBatchReader>
 reader,
+                                             std::shared_ptr<FileReadAhead> 
read_ahead,
+                                             WarmupLevel level,
+                                             
std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      read_ahead_(std::move(read_ahead)),
+      level_(level),
+      arrow_pool_(std::move(arrow_pool)) {}
+
+WarmupFileBatchReader::~WarmupFileBatchReader() {
+    ResetWarmup();
+    reader_->SetPreBufferRangeCallback(nullptr);
+    read_ahead_->ReleaseBuffers();
+}
+
+void WarmupFileBatchReader::Join() const {
+    if (thread_) {
+        thread_->join();
+        thread_.reset();
+    }
+}
+
+void WarmupFileBatchReader::ResetWarmup() {
+    Join();
+    if (batch_ && batch_->ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch_.value()).value().first);
+    }
+    batch_.reset();
+    prepared_ = false;
+    started_ = false;
+    batch_returned_ = false;
+    read_status_ = Status::OK();
+}
+
+void WarmupFileBatchReader::Prepare() {
+    prepared_ = true;
+}
+
+void WarmupFileBatchReader::Warmup() {
+    if (!prepared_ || closed_ || started_ || level_ == WarmupLevel::NONE) {
+        return;
+    }
+    if (level_ == WarmupLevel::RAW) {
+        (void)read_ahead_->Warmup(reader_.get());
+    } else {
+        started_ = true;
+        thread_ = std::make_unique<std::thread>([this]() { 
batch_.emplace(ReadOneBatch()); });
+    }
+}
+
+Result<BatchReader::ReadBatchWithBitmap> WarmupFileBatchReader::ReadOneBatch() 
{
+    PAIMON_RETURN_NOT_OK(read_ahead_->Warmup(reader_.get()));

Review Comment:
   This makes the synchronous reader retain consumed byte ranges until close. 
With a 2 MiB pre-buffer limit, I measured 2 MiB allocated after warmup, 32 MiB 
after sequentially reading 32 MiB, and zero after releasing the cache. The 
limit bounds each fetch window, while previously fetched buffers stay resident. 
This now affects default scans, including `WarmupLevel::NONE` on the first 
actual read. Could the single-reader path evict consumed ranges or enforce a 
resident-byte budget? A scan several times larger than the window would make a 
good peak-memory regression test.



##########
src/paimon/common/reader/warmup_file_batch_reader.cpp:
##########
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+
+namespace paimon {
+
+WarmupFileBatchReader::WarmupFileBatchReader(std::shared_ptr<PrefetchFileBatchReader>
 reader,
+                                             std::shared_ptr<FileReadAhead> 
read_ahead,
+                                             WarmupLevel level,
+                                             
std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      read_ahead_(std::move(read_ahead)),
+      level_(level),
+      arrow_pool_(std::move(arrow_pool)) {}
+
+WarmupFileBatchReader::~WarmupFileBatchReader() {
+    ResetWarmup();
+    reader_->SetPreBufferRangeCallback(nullptr);
+    read_ahead_->ReleaseBuffers();
+}
+
+void WarmupFileBatchReader::Join() const {
+    if (thread_) {
+        thread_->join();
+        thread_.reset();
+    }
+}
+
+void WarmupFileBatchReader::ResetWarmup() {
+    Join();
+    if (batch_ && batch_->ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch_.value()).value().first);
+    }
+    batch_.reset();
+    prepared_ = false;
+    started_ = false;
+    batch_returned_ = false;
+    read_status_ = Status::OK();
+}
+
+void WarmupFileBatchReader::Prepare() {
+    prepared_ = true;
+}
+
+void WarmupFileBatchReader::Warmup() {
+    if (!prepared_ || closed_ || started_ || level_ == WarmupLevel::NONE) {
+        return;
+    }
+    if (level_ == WarmupLevel::RAW) {
+        (void)read_ahead_->Warmup(reader_.get());
+    } else {
+        started_ = true;
+        thread_ = std::make_unique<std::thread>([this]() { 
batch_.emplace(ReadOneBatch()); });
+    }
+}
+
+Result<BatchReader::ReadBatchWithBitmap> WarmupFileBatchReader::ReadOneBatch() 
{
+    PAIMON_RETURN_NOT_OK(read_ahead_->Warmup(reader_.get()));
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, 
reader_->NextBatchWithBitmap());
+    Status status = read_ahead_->GetStatus();
+    if (!status.ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch.first));
+        return status;
+    }
+    return batch;
+}
+
+Result<BatchReader::ReadBatchWithBitmap> 
WarmupFileBatchReader::NextBatchWithBitmap() {
+    if (closed_) {
+        return Status::Invalid("cannot read a closed warmup reader");
+    }
+    PAIMON_RETURN_NOT_OK(read_status_);
+    // A normal read is synchronous unless Warmup() explicitly prepared the 
first batch.
+    started_ = true;
+    Join();
+    batch_returned_ = false;
+    Result<ReadBatchWithBitmap> result = batch_ ? std::move(batch_.value()) : 
ReadOneBatch();
+    batch_.reset();
+    if (result.ok() && !BatchReader::IsEofBatch(result.value())) {
+        batch_returned_ = true;
+    } else if (!result.ok()) {
+        read_status_ = result.status();
+    }
+    return result;
+}
+
+Result<BatchReader::ReadBatch> WarmupFileBatchReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, NextBatchWithBitmap());
+    return ReaderUtils::ApplyBitmapToReadBatch(std::move(batch), arrow_pool_);
+}
+
+Status WarmupFileBatchReader::SetReadSchema(::ArrowSchema* schema,
+                                            const std::shared_ptr<Predicate>& 
predicate,
+                                            const 
std::optional<RoaringBitmap32>& bitmap) {
+    ResetWarmup();
+    if (closed_) {
+        return Status::Invalid("cannot configure a closed warmup reader");
+    }
+    read_ahead_->Reset();
+    read_status_ = reader_->SetReadSchema(schema, predicate, bitmap);
+    PAIMON_RETURN_NOT_OK(read_status_);
+    Prepare();
+    return Status::OK();
+}
+
+std::shared_ptr<Metrics> WarmupFileBatchReader::GetReaderMetrics() const {
+    // The underlying reader may be decoding. Do not inspect its mutable 
metrics until it stops.
+    // Joining is reserved for operations needing the format reader, so a live 
snapshot contains
+    // only the thread-safe cache and I/O counters while the first batch is 
being prepared.
+    auto metrics = std::make_shared<MetricsImpl>();
+    if (!thread_) {

Review Comment:
   `thread_` stays set after decoding finishes until a later call joins it. I 
verified that a format counter visible before `Warmup()` becomes unavailable 
even after the first decode completes, then reappears when the consumer takes 
the batch. An aggregate can consequently drop while progress is being reported. 
Please retain the last safe format-metrics snapshot while decoding and refresh 
it once the task has completed.



##########
src/paimon/common/reader/file_read_ahead.cpp:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/file_read_ahead.h"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <future>
+#include <utility>
+
+#include "paimon/common/executor/future.h"
+#include "paimon/common/io/cache_input_stream.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/format/reader_builder.h"
+
+namespace paimon {
+
+struct PrefetchIoMetricsState {
+    std::atomic<uint64_t> read_requests{0};
+    std::atomic<uint64_t> read_requested_bytes{0};
+    std::atomic<uint64_t> read_physical_bytes{0};
+    std::atomic<uint64_t> read_failed{0};
+    std::atomic<uint64_t> read_latency_count{0};
+    std::atomic<uint64_t> read_latency_sum_us{0};
+    std::atomic<uint64_t> async_requests{0};
+    std::atomic<uint64_t> async_requested_bytes{0};
+    std::atomic<uint64_t> async_physical_bytes{0};
+    std::atomic<uint64_t> async_completed{0};
+    std::atomic<uint64_t> async_failed{0};
+    std::atomic<uint64_t> async_pending{0};
+    std::atomic<uint64_t> async_latency_count{0};
+    std::atomic<uint64_t> async_latency_sum_us{0};
+};
+
+namespace {
+
+constexpr std::memory_order kMetricsMemoryOrder = std::memory_order_relaxed;
+
+template <typename Func>
+auto RunOrSubmit(Executor* executor, Func&& task) -> 
std::future<decltype(task())> {
+    if (executor) {
+        return Via(executor, std::forward<Func>(task));
+    }
+    std::promise<decltype(task())> promise;
+    auto future = promise.get_future();
+    promise.set_value(task());
+    return future;
+}
+
+uint64_t ElapsedMicros(const std::chrono::steady_clock::time_point& start) {
+    return 
static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
+                                     std::chrono::steady_clock::now() - start)
+                                     .count());
+}
+
+void RecordLatency(uint64_t latency_us, std::atomic<uint64_t>* count,
+                   std::atomic<uint64_t>* sum_us) {
+    count->fetch_add(1, kMetricsMemoryOrder);
+    sum_us->fetch_add(latency_us, kMetricsMemoryOrder);
+}
+
+class MetricsInputStream : public InputStream {
+ public:
+    MetricsInputStream(const std::shared_ptr<InputStream>& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(stream), metrics_(metrics) {}
+
+    MetricsInputStream(std::unique_ptr<InputStream>&& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(std::move(stream)), metrics_(metrics) {}
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return stream_->Seek(offset, origin);
+    }
+
+    Result<int64_t> GetPos() const override {
+        return stream_->GetPos();
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size); }, size);
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size, offset); 
}, size);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        metrics_->async_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->async_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                  kMetricsMemoryOrder);
+        metrics_->async_pending.fetch_add(1, kMetricsMemoryOrder);
+        std::shared_ptr<PrefetchIoMetricsState> metrics = metrics_;
+        const auto start = std::chrono::steady_clock::now();
+        stream_->ReadAsync(
+            buffer, size, offset,
+            [metrics, size, start, callback = std::move(callback)](Status 
status) mutable {
+                metrics->async_pending.fetch_sub(1, kMetricsMemoryOrder);
+                if (status.ok()) {
+                    metrics->async_completed.fetch_add(1, kMetricsMemoryOrder);
+                    metrics->async_physical_bytes.fetch_add(
+                        static_cast<uint64_t>(std::max<int64_t>(0, size)), 
kMetricsMemoryOrder);
+                } else {
+                    metrics->async_failed.fetch_add(1, kMetricsMemoryOrder);
+                }
+                RecordLatency(ElapsedMicros(start), 
&metrics->async_latency_count,
+                              &metrics->async_latency_sum_us);
+                callback(status);
+            });
+    }
+
+    Status Close() override {
+        return stream_->Close();
+    }
+
+    Result<std::string> GetUri() const override {
+        return stream_->GetUri();
+    }
+
+    Result<int64_t> Length() const override {
+        return stream_->Length();
+    }
+
+ private:
+    template <typename ReadFunction>
+    Result<int64_t> RecordRead(ReadFunction&& read, int64_t size) {
+        metrics_->read_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->read_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                 kMetricsMemoryOrder);
+        const auto start = std::chrono::steady_clock::now();
+        Result<int64_t> result = read();
+        if (result.ok()) {
+            metrics_->read_physical_bytes.fetch_add(
+                static_cast<uint64_t>(std::max<int64_t>(0, result.value())), 
kMetricsMemoryOrder);
+        } else {
+            metrics_->read_failed.fetch_add(1, kMetricsMemoryOrder);
+        }
+        RecordLatency(ElapsedMicros(start), &metrics_->read_latency_count,
+                      &metrics_->read_latency_sum_us);
+        return result;
+    }
+
+    std::shared_ptr<InputStream> stream_;
+    std::shared_ptr<PrefetchIoMetricsState> metrics_;
+};
+
+}  // namespace
+
+Result<FileReadAhead::OpenedReaders> FileReadAhead::Open(
+    const std::string& path, int64_t file_size, const ReaderBuilder* builder,
+    const std::shared_ptr<FileSystem>& fs, uint32_t reader_count,
+    const std::shared_ptr<Executor>& executor, bool cache_enabled, const 
CacheConfig& cache_config,
+    bool enable_io_metrics, const std::shared_ptr<MemoryPool>& pool) {
+    if (file_size < 0 || !builder || !fs || reader_count == 0 || (!executor && 
reader_count > 1)) {
+        return Status::Invalid("invalid file read-ahead open arguments");
+    }
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics;
+    if (enable_io_metrics) {
+        io_metrics = std::make_shared<PrefetchIoMetricsState>();
+    }
+    // Synchronous reads need no executor and open/build inline. With an 
executor, opening a file
+    // mostly waits on remote I/O, so the streams are all opened concurrently
+    // instead of opening the cache's stream first and paying its round trip 
before the readers'
+    // begin. Both waves run on the read context's `executor`, which is sized 
to the prefetch
+    // parallelism, so no separate pool is spun up and torn down per file and 
the open/build
+    // concurrency follows the configured parallelism instead of a fixed 
default. That reuse is
+    // safe: neither wave's tasks wait on one another or on `executor`, and 
Create never runs on
+    // an `executor` worker, so the CollectAll calls below only ever wait on 
tasks that progress.
+    // The build has to be a second wave rather than folded into the first: 
each build task wraps
+    // its stream in the cache, and a task waiting on the cache would block a 
worker while the
+    // cache's own open is still queued behind it. Wave one only opens, so its 
tasks never wait on
+    // one another.
+    const uint32_t open_count = cache_enabled ? reader_count + 1 : 
reader_count;
+    std::vector<std::future<Result<std::unique_ptr<InputStream>>>> 
open_futures;
+    open_futures.reserve(open_count);
+    for (uint32_t i = 0; i < open_count; i++) {
+        open_futures.push_back(RunOrSubmit(
+            executor.get(), [&fs, &path, file_size]() -> 
Result<std::unique_ptr<InputStream>> {
+                return fs->Open(FileStatus(path, file_size));
+            }));
+    }
+    // The tasks only reference locals of this frame, which stay alive because 
both CollectAll
+    // calls below drain every future before returning.
+    std::vector<Result<std::unique_ptr<InputStream>>> opened_streams = 
CollectAll(open_futures);
+    std::vector<std::unique_ptr<InputStream>> streams;
+    streams.reserve(opened_streams.size());
+    for (auto& opened_stream : opened_streams) {
+        if (!opened_stream.ok()) {
+            return opened_stream.status();
+        }
+        streams.push_back(std::move(opened_stream).value());
+    }
+
+    size_t next_stream = 0;
+    std::shared_ptr<ReadAheadCache> cache;
+    if (cache_enabled) {
+        std::shared_ptr<InputStream> input_stream = 
std::move(streams[next_stream++]);
+        if (io_metrics) {
+            input_stream = std::make_shared<MetricsInputStream>(input_stream, 
io_metrics);
+        }
+        // The file size lets the cache align its blocks to the end of the 
file,
+        // where the metadata the readers read before any range is registered
+        // lives. A zero size means unknown and disables the block cache.
+        cache = std::make_shared<ReadAheadCache>(input_stream, cache_config,
+                                                 
static_cast<uint64_t>(file_size), pool);
+    }
+    // Wave two builds the readers, which reads each file's footer and so 
still needs the cache
+    // above to exist. The builds are concurrent, like the opens were.
+    std::vector<std::future<Result<std::unique_ptr<FileBatchReader>>>> futures;
+    futures.reserve(reader_count);
+    for (uint32_t i = 0; i < reader_count; i++) {
+        futures.push_back(RunOrSubmit(
+            executor.get(),
+            [&builder, &cache, &streams, i, next_stream,
+             io_metrics]() -> Result<std::unique_ptr<FileBatchReader>> {
+                std::unique_ptr<InputStream> input_stream = 
std::move(streams[next_stream + i]);
+                if (io_metrics) {
+                    input_stream =
+                        
std::make_unique<MetricsInputStream>(std::move(input_stream), io_metrics);
+                }
+                auto cache_input_stream =
+                    
std::make_shared<CacheInputStream>(std::move(input_stream), cache);
+                return builder->Build(cache_input_stream);
+            }));
+    }
+    std::vector<std::shared_ptr<PrefetchFileBatchReader>> readers;
+    for (auto& file_batch_reader : CollectAll(futures)) {
+        if (!file_batch_reader.ok()) {
+            return file_batch_reader.status();
+        }
+        std::shared_ptr<FileBatchReader> reader = 
std::move(file_batch_reader).value();
+        auto prefetch_file_batch_reader =
+            std::dynamic_pointer_cast<PrefetchFileBatchReader>(reader);
+        if (prefetch_file_batch_reader == nullptr) {
+            return Status::Invalid(
+                "failed to cast to prefetch file batch reader. file format not 
support prefetch");
+        }
+        readers.emplace_back(prefetch_file_batch_reader);
+    }
+    auto state = std::shared_ptr<FileReadAhead>(new FileReadAhead(cache, 
io_metrics));
+    for (const auto& reader : readers) {
+        std::weak_ptr<FileReadAhead> weak_state = state;
+        reader->SetPreBufferRangeCallback(
+            [weak_state](std::vector<std::pair<uint64_t, uint64_t>>&& ranges) {
+                if (auto shared_state = weak_state.lock()) {
+                    shared_state->RegisterLateRanges(std::move(ranges));
+                }
+            });
+    }
+    return OpenedReaders{std::move(readers), std::move(state)};
+}
+
+Status FileReadAhead::Warmup(PrefetchFileBatchReader* reader) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!cache_ || warmed_) {
+        return status_;
+    }
+    warmed_ = true;
+    auto ranges = reader->PreBufferRange();
+    if (!ranges.ok()) {
+        status_ = ranges.status();
+        ++error_count_;
+        return status_;
+    }
+    std::vector<ByteRange> byte_ranges;
+    for (const auto& range : ranges.value()) {
+        byte_ranges.emplace_back(range.first, range.second);
+    }
+    auto added = cache_->AddRanges(std::move(byte_ranges));

Review Comment:
   Please validate the ranges against the known file size before passing them 
to `AddRanges()`, both here and in `RegisterLateRanges()`. A cache for a 1 KiB 
file accepted a 32 MiB range starting at offset 512 in my test. A corrupt 
Parquet `total_compressed_size` can fit in `int64_t` but cause the coalescer to 
allocate a large number of entries before any read reports EOF. Checking 
`offset <= file_size` and `length <= file_size - offset` before splitting also 
gives us a place to report the file and offending range.



##########
src/paimon/common/reader/file_read_ahead.cpp:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/file_read_ahead.h"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <future>
+#include <utility>
+
+#include "paimon/common/executor/future.h"
+#include "paimon/common/io/cache_input_stream.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/format/reader_builder.h"
+
+namespace paimon {
+
+struct PrefetchIoMetricsState {
+    std::atomic<uint64_t> read_requests{0};
+    std::atomic<uint64_t> read_requested_bytes{0};
+    std::atomic<uint64_t> read_physical_bytes{0};
+    std::atomic<uint64_t> read_failed{0};
+    std::atomic<uint64_t> read_latency_count{0};
+    std::atomic<uint64_t> read_latency_sum_us{0};
+    std::atomic<uint64_t> async_requests{0};
+    std::atomic<uint64_t> async_requested_bytes{0};
+    std::atomic<uint64_t> async_physical_bytes{0};
+    std::atomic<uint64_t> async_completed{0};
+    std::atomic<uint64_t> async_failed{0};
+    std::atomic<uint64_t> async_pending{0};
+    std::atomic<uint64_t> async_latency_count{0};
+    std::atomic<uint64_t> async_latency_sum_us{0};
+};
+
+namespace {
+
+constexpr std::memory_order kMetricsMemoryOrder = std::memory_order_relaxed;
+
+template <typename Func>
+auto RunOrSubmit(Executor* executor, Func&& task) -> 
std::future<decltype(task())> {
+    if (executor) {
+        return Via(executor, std::forward<Func>(task));
+    }
+    std::promise<decltype(task())> promise;
+    auto future = promise.get_future();
+    promise.set_value(task());
+    return future;
+}
+
+uint64_t ElapsedMicros(const std::chrono::steady_clock::time_point& start) {
+    return 
static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
+                                     std::chrono::steady_clock::now() - start)
+                                     .count());
+}
+
+void RecordLatency(uint64_t latency_us, std::atomic<uint64_t>* count,
+                   std::atomic<uint64_t>* sum_us) {
+    count->fetch_add(1, kMetricsMemoryOrder);
+    sum_us->fetch_add(latency_us, kMetricsMemoryOrder);
+}
+
+class MetricsInputStream : public InputStream {
+ public:
+    MetricsInputStream(const std::shared_ptr<InputStream>& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(stream), metrics_(metrics) {}
+
+    MetricsInputStream(std::unique_ptr<InputStream>&& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(std::move(stream)), metrics_(metrics) {}
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return stream_->Seek(offset, origin);
+    }
+
+    Result<int64_t> GetPos() const override {
+        return stream_->GetPos();
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size); }, size);
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size, offset); 
}, size);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        metrics_->async_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->async_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                  kMetricsMemoryOrder);
+        metrics_->async_pending.fetch_add(1, kMetricsMemoryOrder);
+        std::shared_ptr<PrefetchIoMetricsState> metrics = metrics_;
+        const auto start = std::chrono::steady_clock::now();
+        stream_->ReadAsync(
+            buffer, size, offset,
+            [metrics, size, start, callback = std::move(callback)](Status 
status) mutable {
+                metrics->async_pending.fetch_sub(1, kMetricsMemoryOrder);
+                if (status.ok()) {
+                    metrics->async_completed.fetch_add(1, kMetricsMemoryOrder);
+                    metrics->async_physical_bytes.fetch_add(
+                        static_cast<uint64_t>(std::max<int64_t>(0, size)), 
kMetricsMemoryOrder);
+                } else {
+                    metrics->async_failed.fetch_add(1, kMetricsMemoryOrder);
+                }
+                RecordLatency(ElapsedMicros(start), 
&metrics->async_latency_count,
+                              &metrics->async_latency_sum_us);
+                callback(status);
+            });
+    }
+
+    Status Close() override {
+        return stream_->Close();
+    }
+
+    Result<std::string> GetUri() const override {
+        return stream_->GetUri();
+    }
+
+    Result<int64_t> Length() const override {
+        return stream_->Length();
+    }
+
+ private:
+    template <typename ReadFunction>
+    Result<int64_t> RecordRead(ReadFunction&& read, int64_t size) {
+        metrics_->read_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->read_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                 kMetricsMemoryOrder);
+        const auto start = std::chrono::steady_clock::now();
+        Result<int64_t> result = read();
+        if (result.ok()) {
+            metrics_->read_physical_bytes.fetch_add(
+                static_cast<uint64_t>(std::max<int64_t>(0, result.value())), 
kMetricsMemoryOrder);
+        } else {
+            metrics_->read_failed.fetch_add(1, kMetricsMemoryOrder);
+        }
+        RecordLatency(ElapsedMicros(start), &metrics_->read_latency_count,
+                      &metrics_->read_latency_sum_us);
+        return result;
+    }
+
+    std::shared_ptr<InputStream> stream_;
+    std::shared_ptr<PrefetchIoMetricsState> metrics_;
+};
+
+}  // namespace
+
+Result<FileReadAhead::OpenedReaders> FileReadAhead::Open(
+    const std::string& path, int64_t file_size, const ReaderBuilder* builder,
+    const std::shared_ptr<FileSystem>& fs, uint32_t reader_count,
+    const std::shared_ptr<Executor>& executor, bool cache_enabled, const 
CacheConfig& cache_config,
+    bool enable_io_metrics, const std::shared_ptr<MemoryPool>& pool) {
+    if (file_size < 0 || !builder || !fs || reader_count == 0 || (!executor && 
reader_count > 1)) {
+        return Status::Invalid("invalid file read-ahead open arguments");
+    }
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics;
+    if (enable_io_metrics) {
+        io_metrics = std::make_shared<PrefetchIoMetricsState>();
+    }
+    // Synchronous reads need no executor and open/build inline. With an 
executor, opening a file
+    // mostly waits on remote I/O, so the streams are all opened concurrently
+    // instead of opening the cache's stream first and paying its round trip 
before the readers'
+    // begin. Both waves run on the read context's `executor`, which is sized 
to the prefetch
+    // parallelism, so no separate pool is spun up and torn down per file and 
the open/build
+    // concurrency follows the configured parallelism instead of a fixed 
default. That reuse is
+    // safe: neither wave's tasks wait on one another or on `executor`, and 
Create never runs on
+    // an `executor` worker, so the CollectAll calls below only ever wait on 
tasks that progress.
+    // The build has to be a second wave rather than folded into the first: 
each build task wraps
+    // its stream in the cache, and a task waiting on the cache would block a 
worker while the
+    // cache's own open is still queued behind it. Wave one only opens, so its 
tasks never wait on
+    // one another.
+    const uint32_t open_count = cache_enabled ? reader_count + 1 : 
reader_count;
+    std::vector<std::future<Result<std::unique_ptr<InputStream>>>> 
open_futures;
+    open_futures.reserve(open_count);
+    for (uint32_t i = 0; i < open_count; i++) {
+        open_futures.push_back(RunOrSubmit(
+            executor.get(), [&fs, &path, file_size]() -> 
Result<std::unique_ptr<InputStream>> {
+                return fs->Open(FileStatus(path, file_size));
+            }));
+    }
+    // The tasks only reference locals of this frame, which stay alive because 
both CollectAll
+    // calls below drain every future before returning.
+    std::vector<Result<std::unique_ptr<InputStream>>> opened_streams = 
CollectAll(open_futures);
+    std::vector<std::unique_ptr<InputStream>> streams;
+    streams.reserve(opened_streams.size());
+    for (auto& opened_stream : opened_streams) {
+        if (!opened_stream.ok()) {
+            return opened_stream.status();
+        }
+        streams.push_back(std::move(opened_stream).value());
+    }
+
+    size_t next_stream = 0;
+    std::shared_ptr<ReadAheadCache> cache;
+    if (cache_enabled) {
+        std::shared_ptr<InputStream> input_stream = 
std::move(streams[next_stream++]);
+        if (io_metrics) {
+            input_stream = std::make_shared<MetricsInputStream>(input_stream, 
io_metrics);
+        }
+        // The file size lets the cache align its blocks to the end of the 
file,
+        // where the metadata the readers read before any range is registered
+        // lives. A zero size means unknown and disables the block cache.
+        cache = std::make_shared<ReadAheadCache>(input_stream, cache_config,
+                                                 
static_cast<uint64_t>(file_size), pool);
+    }
+    // Wave two builds the readers, which reads each file's footer and so 
still needs the cache
+    // above to exist. The builds are concurrent, like the opens were.
+    std::vector<std::future<Result<std::unique_ptr<FileBatchReader>>>> futures;
+    futures.reserve(reader_count);
+    for (uint32_t i = 0; i < reader_count; i++) {
+        futures.push_back(RunOrSubmit(
+            executor.get(),
+            [&builder, &cache, &streams, i, next_stream,
+             io_metrics]() -> Result<std::unique_ptr<FileBatchReader>> {
+                std::unique_ptr<InputStream> input_stream = 
std::move(streams[next_stream + i]);
+                if (io_metrics) {
+                    input_stream =
+                        
std::make_unique<MetricsInputStream>(std::move(input_stream), io_metrics);
+                }
+                auto cache_input_stream =
+                    
std::make_shared<CacheInputStream>(std::move(input_stream), cache);
+                return builder->Build(cache_input_stream);
+            }));
+    }
+    std::vector<std::shared_ptr<PrefetchFileBatchReader>> readers;
+    for (auto& file_batch_reader : CollectAll(futures)) {
+        if (!file_batch_reader.ok()) {
+            return file_batch_reader.status();
+        }
+        std::shared_ptr<FileBatchReader> reader = 
std::move(file_batch_reader).value();
+        auto prefetch_file_batch_reader =
+            std::dynamic_pointer_cast<PrefetchFileBatchReader>(reader);
+        if (prefetch_file_batch_reader == nullptr) {
+            return Status::Invalid(
+                "failed to cast to prefetch file batch reader. file format not 
support prefetch");
+        }
+        readers.emplace_back(prefetch_file_batch_reader);
+    }
+    auto state = std::shared_ptr<FileReadAhead>(new FileReadAhead(cache, 
io_metrics));
+    for (const auto& reader : readers) {
+        std::weak_ptr<FileReadAhead> weak_state = state;
+        reader->SetPreBufferRangeCallback(
+            [weak_state](std::vector<std::pair<uint64_t, uint64_t>>&& ranges) {
+                if (auto shared_state = weak_state.lock()) {
+                    shared_state->RegisterLateRanges(std::move(ranges));
+                }
+            });
+    }
+    return OpenedReaders{std::move(readers), std::move(state)};
+}
+
+Status FileReadAhead::Warmup(PrefetchFileBatchReader* reader) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!cache_ || warmed_) {
+        return status_;
+    }
+    warmed_ = true;
+    auto ranges = reader->PreBufferRange();

Review Comment:
   On ORC, this call enumerates every stripe and reads and parses its footer 
synchronously before dispatching the first cache fetch. With default RAW 
warmup, a remote file with many stripes can therefore delay the first batch, 
even for `LIMIT 1` on a filesystem with asynchronous reads. Could range 
discovery start with the first stripe or pre-buffer window and advance on 
demand? A counting stream could assert which stripe footers were touched before 
the first batch was returned.



##########
src/paimon/common/reader/file_read_ahead.cpp:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/file_read_ahead.h"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <future>
+#include <utility>
+
+#include "paimon/common/executor/future.h"
+#include "paimon/common/io/cache_input_stream.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/format/reader_builder.h"
+
+namespace paimon {
+
+struct PrefetchIoMetricsState {
+    std::atomic<uint64_t> read_requests{0};
+    std::atomic<uint64_t> read_requested_bytes{0};
+    std::atomic<uint64_t> read_physical_bytes{0};
+    std::atomic<uint64_t> read_failed{0};
+    std::atomic<uint64_t> read_latency_count{0};
+    std::atomic<uint64_t> read_latency_sum_us{0};
+    std::atomic<uint64_t> async_requests{0};
+    std::atomic<uint64_t> async_requested_bytes{0};
+    std::atomic<uint64_t> async_physical_bytes{0};
+    std::atomic<uint64_t> async_completed{0};
+    std::atomic<uint64_t> async_failed{0};
+    std::atomic<uint64_t> async_pending{0};
+    std::atomic<uint64_t> async_latency_count{0};
+    std::atomic<uint64_t> async_latency_sum_us{0};
+};
+
+namespace {
+
+constexpr std::memory_order kMetricsMemoryOrder = std::memory_order_relaxed;
+
+template <typename Func>
+auto RunOrSubmit(Executor* executor, Func&& task) -> 
std::future<decltype(task())> {
+    if (executor) {
+        return Via(executor, std::forward<Func>(task));
+    }
+    std::promise<decltype(task())> promise;
+    auto future = promise.get_future();
+    promise.set_value(task());
+    return future;
+}
+
+uint64_t ElapsedMicros(const std::chrono::steady_clock::time_point& start) {
+    return 
static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
+                                     std::chrono::steady_clock::now() - start)
+                                     .count());
+}
+
+void RecordLatency(uint64_t latency_us, std::atomic<uint64_t>* count,
+                   std::atomic<uint64_t>* sum_us) {
+    count->fetch_add(1, kMetricsMemoryOrder);
+    sum_us->fetch_add(latency_us, kMetricsMemoryOrder);
+}
+
+class MetricsInputStream : public InputStream {
+ public:
+    MetricsInputStream(const std::shared_ptr<InputStream>& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(stream), metrics_(metrics) {}
+
+    MetricsInputStream(std::unique_ptr<InputStream>&& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(std::move(stream)), metrics_(metrics) {}
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return stream_->Seek(offset, origin);
+    }
+
+    Result<int64_t> GetPos() const override {
+        return stream_->GetPos();
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size); }, size);
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size, offset); 
}, size);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        metrics_->async_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->async_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                  kMetricsMemoryOrder);
+        metrics_->async_pending.fetch_add(1, kMetricsMemoryOrder);
+        std::shared_ptr<PrefetchIoMetricsState> metrics = metrics_;
+        const auto start = std::chrono::steady_clock::now();
+        stream_->ReadAsync(
+            buffer, size, offset,
+            [metrics, size, start, callback = std::move(callback)](Status 
status) mutable {
+                metrics->async_pending.fetch_sub(1, kMetricsMemoryOrder);
+                if (status.ok()) {
+                    metrics->async_completed.fetch_add(1, kMetricsMemoryOrder);
+                    metrics->async_physical_bytes.fetch_add(
+                        static_cast<uint64_t>(std::max<int64_t>(0, size)), 
kMetricsMemoryOrder);
+                } else {
+                    metrics->async_failed.fetch_add(1, kMetricsMemoryOrder);
+                }
+                RecordLatency(ElapsedMicros(start), 
&metrics->async_latency_count,
+                              &metrics->async_latency_sum_us);
+                callback(status);
+            });
+    }
+
+    Status Close() override {
+        return stream_->Close();
+    }
+
+    Result<std::string> GetUri() const override {
+        return stream_->GetUri();
+    }
+
+    Result<int64_t> Length() const override {
+        return stream_->Length();
+    }
+
+ private:
+    template <typename ReadFunction>
+    Result<int64_t> RecordRead(ReadFunction&& read, int64_t size) {
+        metrics_->read_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->read_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                 kMetricsMemoryOrder);
+        const auto start = std::chrono::steady_clock::now();
+        Result<int64_t> result = read();
+        if (result.ok()) {
+            metrics_->read_physical_bytes.fetch_add(
+                static_cast<uint64_t>(std::max<int64_t>(0, result.value())), 
kMetricsMemoryOrder);
+        } else {
+            metrics_->read_failed.fetch_add(1, kMetricsMemoryOrder);
+        }
+        RecordLatency(ElapsedMicros(start), &metrics_->read_latency_count,
+                      &metrics_->read_latency_sum_us);
+        return result;
+    }
+
+    std::shared_ptr<InputStream> stream_;
+    std::shared_ptr<PrefetchIoMetricsState> metrics_;
+};
+
+}  // namespace
+
+Result<FileReadAhead::OpenedReaders> FileReadAhead::Open(
+    const std::string& path, int64_t file_size, const ReaderBuilder* builder,
+    const std::shared_ptr<FileSystem>& fs, uint32_t reader_count,
+    const std::shared_ptr<Executor>& executor, bool cache_enabled, const 
CacheConfig& cache_config,
+    bool enable_io_metrics, const std::shared_ptr<MemoryPool>& pool) {
+    if (file_size < 0 || !builder || !fs || reader_count == 0 || (!executor && 
reader_count > 1)) {
+        return Status::Invalid("invalid file read-ahead open arguments");
+    }
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics;
+    if (enable_io_metrics) {
+        io_metrics = std::make_shared<PrefetchIoMetricsState>();
+    }
+    // Synchronous reads need no executor and open/build inline. With an 
executor, opening a file
+    // mostly waits on remote I/O, so the streams are all opened concurrently
+    // instead of opening the cache's stream first and paying its round trip 
before the readers'
+    // begin. Both waves run on the read context's `executor`, which is sized 
to the prefetch
+    // parallelism, so no separate pool is spun up and torn down per file and 
the open/build
+    // concurrency follows the configured parallelism instead of a fixed 
default. That reuse is
+    // safe: neither wave's tasks wait on one another or on `executor`, and 
Create never runs on
+    // an `executor` worker, so the CollectAll calls below only ever wait on 
tasks that progress.
+    // The build has to be a second wave rather than folded into the first: 
each build task wraps
+    // its stream in the cache, and a task waiting on the cache would block a 
worker while the
+    // cache's own open is still queued behind it. Wave one only opens, so its 
tasks never wait on
+    // one another.
+    const uint32_t open_count = cache_enabled ? reader_count + 1 : 
reader_count;
+    std::vector<std::future<Result<std::unique_ptr<InputStream>>>> 
open_futures;
+    open_futures.reserve(open_count);
+    for (uint32_t i = 0; i < open_count; i++) {
+        open_futures.push_back(RunOrSubmit(
+            executor.get(), [&fs, &path, file_size]() -> 
Result<std::unique_ptr<InputStream>> {
+                return fs->Open(FileStatus(path, file_size));
+            }));
+    }
+    // The tasks only reference locals of this frame, which stay alive because 
both CollectAll
+    // calls below drain every future before returning.
+    std::vector<Result<std::unique_ptr<InputStream>>> opened_streams = 
CollectAll(open_futures);
+    std::vector<std::unique_ptr<InputStream>> streams;
+    streams.reserve(opened_streams.size());
+    for (auto& opened_stream : opened_streams) {
+        if (!opened_stream.ok()) {
+            return opened_stream.status();
+        }
+        streams.push_back(std::move(opened_stream).value());
+    }
+
+    size_t next_stream = 0;
+    std::shared_ptr<ReadAheadCache> cache;
+    if (cache_enabled) {
+        std::shared_ptr<InputStream> input_stream = 
std::move(streams[next_stream++]);
+        if (io_metrics) {
+            input_stream = std::make_shared<MetricsInputStream>(input_stream, 
io_metrics);
+        }
+        // The file size lets the cache align its blocks to the end of the 
file,
+        // where the metadata the readers read before any range is registered
+        // lives. A zero size means unknown and disables the block cache.
+        cache = std::make_shared<ReadAheadCache>(input_stream, cache_config,
+                                                 
static_cast<uint64_t>(file_size), pool);
+    }
+    // Wave two builds the readers, which reads each file's footer and so 
still needs the cache
+    // above to exist. The builds are concurrent, like the opens were.
+    std::vector<std::future<Result<std::unique_ptr<FileBatchReader>>>> futures;
+    futures.reserve(reader_count);
+    for (uint32_t i = 0; i < reader_count; i++) {
+        futures.push_back(RunOrSubmit(
+            executor.get(),
+            [&builder, &cache, &streams, i, next_stream,
+             io_metrics]() -> Result<std::unique_ptr<FileBatchReader>> {
+                std::unique_ptr<InputStream> input_stream = 
std::move(streams[next_stream + i]);
+                if (io_metrics) {
+                    input_stream =
+                        
std::make_unique<MetricsInputStream>(std::move(input_stream), io_metrics);
+                }
+                auto cache_input_stream =
+                    
std::make_shared<CacheInputStream>(std::move(input_stream), cache);
+                return builder->Build(cache_input_stream);
+            }));
+    }
+    std::vector<std::shared_ptr<PrefetchFileBatchReader>> readers;
+    for (auto& file_batch_reader : CollectAll(futures)) {
+        if (!file_batch_reader.ok()) {
+            return file_batch_reader.status();
+        }
+        std::shared_ptr<FileBatchReader> reader = 
std::move(file_batch_reader).value();
+        auto prefetch_file_batch_reader =
+            std::dynamic_pointer_cast<PrefetchFileBatchReader>(reader);
+        if (prefetch_file_batch_reader == nullptr) {
+            return Status::Invalid(
+                "failed to cast to prefetch file batch reader. file format not 
support prefetch");
+        }
+        readers.emplace_back(prefetch_file_batch_reader);
+    }
+    auto state = std::shared_ptr<FileReadAhead>(new FileReadAhead(cache, 
io_metrics));
+    for (const auto& reader : readers) {
+        std::weak_ptr<FileReadAhead> weak_state = state;
+        reader->SetPreBufferRangeCallback(

Review Comment:
   Please install this callback only when the shared cache is enabled. 
`LateMaterializingFileBatchReader` uses the presence of the callback to decide 
whether to call `PreBufferRange()` for the payload pass. I reproduced a read 
failure with the cache off and `WarmupLevel::NONE` by failing only that range 
query. The callback's later `!cache_` check is too late. A test that asserts 
zero payload range queries with caching disabled would catch this.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to