SteNicholas commented on code in PR #513:
URL: https://github.com/apache/paimon-cpp/pull/513#discussion_r4480319523


##########
src/paimon/common/reader/warmup_file_batch_reader.cpp:
##########
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+
+namespace paimon {
+
+WarmupFileBatchReader::WarmupFileBatchReader(std::shared_ptr<PrefetchFileBatchReader>
 reader,
+                                             std::shared_ptr<FileReadAhead> 
read_ahead,
+                                             WarmupLevel level,
+                                             
std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      read_ahead_(std::move(read_ahead)),
+      level_(level),
+      arrow_pool_(std::move(arrow_pool)) {}
+
+WarmupFileBatchReader::~WarmupFileBatchReader() {
+    ResetReadState();
+    read_ahead_->ReleaseBuffers();
+}
+
+void WarmupFileBatchReader::ResetReadState() {
+    prepared_ = false;
+    batch_returned_ = false;
+    read_status_ = Status::OK();
+}
+
+void WarmupFileBatchReader::Prepare() {
+    prepared_ = true;
+}
+
+void WarmupFileBatchReader::Warmup() {
+    if (!prepared_ || closed_ || level_ != WarmupLevel::RAW) {
+        return;
+    }
+    (void)read_ahead_->Warmup(reader_.get());
+}
+
+Result<BatchReader::ReadBatchWithBitmap> WarmupFileBatchReader::ReadOneBatch() 
{
+    PAIMON_RETURN_NOT_OK(read_ahead_->Warmup(reader_.get()));
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, 
reader_->NextBatchWithBitmap());
+    Status status = read_ahead_->GetStatus();
+    if (!status.ok()) {
+        ReaderUtils::ReleaseReadBatch(std::move(batch.first));
+        return status;
+    }
+    return batch;
+}
+
+Result<BatchReader::ReadBatchWithBitmap> 
WarmupFileBatchReader::NextBatchWithBitmap() {
+    if (closed_) {
+        return Status::Invalid("cannot read a closed warmup reader");
+    }
+    PAIMON_RETURN_NOT_OK(read_status_);
+    batch_returned_ = false;
+    Result<ReadBatchWithBitmap> result = ReadOneBatch();
+    if (result.ok() && !BatchReader::IsEofBatch(result.value())) {
+        batch_returned_ = true;
+    } else if (!result.ok()) {
+        read_status_ = result.status();
+    }
+    return result;
+}
+
+Result<BatchReader::ReadBatch> WarmupFileBatchReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, NextBatchWithBitmap());
+    return ReaderUtils::ApplyBitmapToReadBatch(std::move(batch), arrow_pool_);

Review Comment:
   Please preserve the file-row mapping when compacting the bitmap here. 
`GetPreviousBatchFileRowId()` still forwards the returned batch's index 
directly to the underlying reader, whose last batch is the unfiltered 
`NextBatchWithBitmap()` result. With file rows `[0, 1, 2, 3]` and bitmap `{1, 
3}`, I reproduced two returned rows mapping to `[0, 1]` instead of `[1, 3]`; 
index 2 was also accepted despite the returned batch having only two rows. The 
same capable reader's own `NextBatch()` and row-ID mapping worked correctly 
without this wrapper. This affects readers/plugins returning sparse bitmaps 
even with caching disabled and `WarmupLevel::NONE`, since the synchronous path 
still adds the wrapper. Could we retain the selected row mapping, or preserve 
the underlying `NextBatch()` semantics, and add sparse-bitmap and out-of-range 
mapping coverage?



##########
src/paimon/common/reader/file_read_ahead.cpp:
##########
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/common/reader/file_read_ahead.h"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <future>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/common/executor/future.h"
+#include "paimon/common/io/cache_input_stream.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/warmup_file_batch_reader.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/format/reader_builder.h"
+
+namespace paimon {
+
+struct PrefetchIoMetricsState {
+    std::atomic<uint64_t> read_requests{0};
+    std::atomic<uint64_t> read_requested_bytes{0};
+    std::atomic<uint64_t> read_physical_bytes{0};
+    std::atomic<uint64_t> read_failed{0};
+    std::atomic<uint64_t> read_latency_count{0};
+    std::atomic<uint64_t> read_latency_sum_us{0};
+    std::atomic<uint64_t> async_requests{0};
+    std::atomic<uint64_t> async_requested_bytes{0};
+    std::atomic<uint64_t> async_physical_bytes{0};
+    std::atomic<uint64_t> async_completed{0};
+    std::atomic<uint64_t> async_failed{0};
+    std::atomic<uint64_t> async_pending{0};
+    std::atomic<uint64_t> async_latency_count{0};
+    std::atomic<uint64_t> async_latency_sum_us{0};
+};
+
+namespace {
+
+constexpr std::memory_order kMetricsMemoryOrder = std::memory_order_relaxed;
+
+template <typename Func>
+auto RunOrSubmit(Executor* executor, Func&& task) -> 
std::future<decltype(task())> {
+    if (executor) {
+        return Via(executor, std::forward<Func>(task));
+    }
+    std::promise<decltype(task())> promise;
+    auto future = promise.get_future();
+    promise.set_value(task());
+    return future;
+}
+
+uint64_t ElapsedMicros(const std::chrono::steady_clock::time_point& start) {
+    return 
static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
+                                     std::chrono::steady_clock::now() - start)
+                                     .count());
+}
+
+void RecordLatency(uint64_t latency_us, std::atomic<uint64_t>* count,
+                   std::atomic<uint64_t>* sum_us) {
+    count->fetch_add(1, kMetricsMemoryOrder);
+    sum_us->fetch_add(latency_us, kMetricsMemoryOrder);
+}
+
+class MetricsInputStream : public InputStream {
+ public:
+    MetricsInputStream(const std::shared_ptr<InputStream>& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(stream), metrics_(metrics) {}
+
+    MetricsInputStream(std::unique_ptr<InputStream>&& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(std::move(stream)), metrics_(metrics) {}
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return stream_->Seek(offset, origin);
+    }
+
+    Result<int64_t> GetPos() const override {
+        return stream_->GetPos();
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size); }, size);
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size, offset); 
}, size);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        metrics_->async_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->async_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                  kMetricsMemoryOrder);
+        metrics_->async_pending.fetch_add(1, kMetricsMemoryOrder);
+        std::shared_ptr<PrefetchIoMetricsState> metrics = metrics_;
+        const auto start = std::chrono::steady_clock::now();
+        stream_->ReadAsync(
+            buffer, size, offset,
+            [metrics, size, start, callback = std::move(callback)](Status 
status) mutable {
+                metrics->async_pending.fetch_sub(1, kMetricsMemoryOrder);
+                if (status.ok()) {
+                    metrics->async_completed.fetch_add(1, kMetricsMemoryOrder);
+                    metrics->async_physical_bytes.fetch_add(
+                        static_cast<uint64_t>(std::max<int64_t>(0, size)), 
kMetricsMemoryOrder);
+                } else {
+                    metrics->async_failed.fetch_add(1, kMetricsMemoryOrder);
+                }
+                RecordLatency(ElapsedMicros(start), 
&metrics->async_latency_count,
+                              &metrics->async_latency_sum_us);
+                callback(status);
+            });
+    }
+
+    Status Close() override {
+        return stream_->Close();
+    }
+
+    Result<std::string> GetUri() const override {
+        return stream_->GetUri();
+    }
+
+    Result<int64_t> Length() const override {
+        return stream_->Length();
+    }
+
+ private:
+    template <typename ReadFunction>
+    Result<int64_t> RecordRead(ReadFunction&& read, int64_t size) {
+        metrics_->read_requests.fetch_add(1, kMetricsMemoryOrder);
+        
metrics_->read_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0,
 size)),
+                                                 kMetricsMemoryOrder);
+        const auto start = std::chrono::steady_clock::now();
+        Result<int64_t> result = read();
+        if (result.ok()) {
+            metrics_->read_physical_bytes.fetch_add(
+                static_cast<uint64_t>(std::max<int64_t>(0, result.value())), 
kMetricsMemoryOrder);
+        } else {
+            metrics_->read_failed.fetch_add(1, kMetricsMemoryOrder);
+        }
+        RecordLatency(ElapsedMicros(start), &metrics_->read_latency_count,
+                      &metrics_->read_latency_sum_us);
+        return result;
+    }
+
+    std::shared_ptr<InputStream> stream_;
+    std::shared_ptr<PrefetchIoMetricsState> metrics_;
+};
+
+}  // namespace
+
+std::shared_ptr<FileReadAhead> FileReadAhead::Create(const 
std::shared_ptr<ReadAheadCache>& cache,
+                                                     const std::string& path) {
+    return std::shared_ptr<FileReadAhead>(new FileReadAhead(cache, path));
+}
+
+Result<FileReadAhead::OpenedReaders> FileReadAhead::Open(
+    const std::string& path, int64_t file_size, const ReaderBuilder* builder,
+    const std::shared_ptr<FileSystem>& fs, uint32_t reader_count,
+    const std::shared_ptr<Executor>& executor, bool cache_enabled, const 
CacheConfig& cache_config,
+    bool enable_io_metrics, const std::shared_ptr<MemoryPool>& pool) {
+    if (file_size < 0 || !builder || !fs || reader_count == 0 || (!executor && 
reader_count > 1)) {
+        return Status::Invalid("invalid file read-ahead open arguments");
+    }
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics;
+    if (enable_io_metrics) {
+        io_metrics = std::make_shared<PrefetchIoMetricsState>();
+    }
+    // Synchronous reads need no executor and open/build inline. With an 
executor, opening a file
+    // mostly waits on remote I/O, so the streams are all opened concurrently
+    // instead of opening the cache's stream first and paying its round trip 
before the readers'
+    // begin. Both waves run on the read context's `executor`, which is sized 
to the prefetch
+    // parallelism, so no separate pool is spun up and torn down per file and 
the open/build
+    // concurrency follows the configured parallelism instead of a fixed 
default. That reuse is
+    // safe: neither wave's tasks wait on one another or on `executor`, and 
Create never runs on
+    // an `executor` worker, so the CollectAll calls below only ever wait on 
tasks that progress.
+    // The build has to be a second wave rather than folded into the first: 
each build task wraps
+    // its stream in the cache, and a task waiting on the cache would block a 
worker while the
+    // cache's own open is still queued behind it. Wave one only opens, so its 
tasks never wait on
+    // one another.
+    const uint32_t open_count = cache_enabled ? reader_count + 1 : 
reader_count;
+    std::vector<std::future<Result<std::unique_ptr<InputStream>>>> 
open_futures;
+    open_futures.reserve(open_count);
+    for (uint32_t i = 0; i < open_count; i++) {
+        open_futures.push_back(RunOrSubmit(
+            executor.get(), [&fs, &path, file_size]() -> 
Result<std::unique_ptr<InputStream>> {
+                return fs->Open(FileStatus(path, file_size));
+            }));
+    }
+    // The tasks only reference locals of this frame, which stay alive because 
both CollectAll
+    // calls below drain every future before returning.
+    std::vector<Result<std::unique_ptr<InputStream>>> opened_streams = 
CollectAll(open_futures);
+    std::vector<std::unique_ptr<InputStream>> streams;
+    streams.reserve(opened_streams.size());
+    for (auto& opened_stream : opened_streams) {
+        if (!opened_stream.ok()) {
+            return opened_stream.status();
+        }
+        streams.push_back(std::move(opened_stream).value());
+    }
+
+    size_t next_stream = 0;
+    std::shared_ptr<ReadAheadCache> cache;
+    if (cache_enabled) {
+        std::shared_ptr<InputStream> input_stream = 
std::move(streams[next_stream++]);
+        if (io_metrics) {
+            input_stream = std::make_shared<MetricsInputStream>(input_stream, 
io_metrics);
+        }
+        // The file size lets the cache align its blocks to the end of the 
file,
+        // where the metadata the readers read before any range is registered
+        // lives. A zero size means unknown and disables the block cache.
+        cache = std::make_shared<ReadAheadCache>(input_stream, cache_config,
+                                                 
static_cast<uint64_t>(file_size), pool);
+    }
+    auto state = Create(cache, path);
+    state->io_metrics_ = io_metrics;
+    // Wave two builds the readers, which reads each file's footer and so 
still needs the cache
+    // above to exist. The builds are concurrent, like the opens were.
+    std::vector<std::future<Result<std::unique_ptr<FileBatchReader>>>> futures;
+    futures.reserve(reader_count);
+    for (uint32_t i = 0; i < reader_count; i++) {
+        futures.push_back(RunOrSubmit(
+            executor.get(),
+            [&builder, &cache, &streams, i, next_stream, io_metrics,
+             state]() -> Result<std::unique_ptr<FileBatchReader>> {
+                std::unique_ptr<InputStream> input_stream = 
std::move(streams[next_stream + i]);
+                if (io_metrics) {
+                    input_stream =
+                        
std::make_unique<MetricsInputStream>(std::move(input_stream), io_metrics);
+                }
+                auto cache_input_stream = std::make_shared<CacheInputStream>(
+                    std::move(input_stream), cache, 
CacheInputStream::CacheFactory(), state);
+                return builder->Build(cache_input_stream);
+            }));
+    }
+    std::vector<std::shared_ptr<PrefetchFileBatchReader>> readers;
+    for (auto& file_batch_reader : CollectAll(futures)) {
+        if (!file_batch_reader.ok()) {
+            return file_batch_reader.status();
+        }
+        std::shared_ptr<FileBatchReader> reader = 
std::move(file_batch_reader).value();
+        auto prefetch_file_batch_reader =
+            std::dynamic_pointer_cast<PrefetchFileBatchReader>(reader);
+        if (prefetch_file_batch_reader == nullptr) {
+            return Status::Invalid(
+                "failed to cast to prefetch file batch reader. file format not 
support prefetch");
+        }
+        readers.emplace_back(prefetch_file_batch_reader);
+    }
+    return OpenedReaders{std::move(readers), std::move(state)};
+}
+
+Result<std::unique_ptr<FileBatchReader>> FileReadAhead::OpenSynchronous(
+    const std::string& path, int64_t file_size, const ReaderBuilder* builder,
+    const std::shared_ptr<FileSystem>& fs, bool cache_enabled, const 
CacheConfig& cache_config,
+    bool enable_io_metrics, WarmupLevel level, const 
std::shared_ptr<MemoryPool>& pool,
+    const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (file_size < 0 || !builder || !fs) {
+        return Status::Invalid("invalid synchronous file read-ahead open 
arguments");
+    }
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics;
+    if (enable_io_metrics) {
+        io_metrics = std::make_shared<PrefetchIoMetricsState>();
+    }
+    auto state = Create(nullptr, path);
+    state->io_metrics_ = io_metrics;
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InputStream> input_stream,
+                           fs->Open(FileStatus(path, file_size)));
+    if (io_metrics) {
+        input_stream = 
std::make_unique<MetricsInputStream>(std::move(input_stream), io_metrics);
+    }
+    CacheInputStream::CacheFactory cache_factory;
+    if (cache_enabled) {
+        cache_factory = [state, fs, path, file_size, cache_config,
+                         pool]() -> Result<std::shared_ptr<ReadAheadCache>> {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> cache_stream,
+                                   fs->Open(FileStatus(path, file_size)));
+            if (state->io_metrics_) {
+                cache_stream =
+                    std::make_shared<MetricsInputStream>(cache_stream, 
state->io_metrics_);
+            }
+            state->cache_ = std::make_shared<ReadAheadCache>(
+                cache_stream, cache_config, static_cast<uint64_t>(file_size), 
pool);
+            return state->cache_;
+        };
+    }
+    auto stream = std::make_shared<CacheInputStream>(std::move(input_stream), 
nullptr,
+                                                     std::move(cache_factory), 
state);
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> reader, 
builder->Build(stream));
+    // The byte-range interface, rather than the format name, determines 
cache/warmup support.
+    if (!dynamic_cast<PrefetchFileBatchReader*>(reader.get())) {
+        stream->ClearCacheFactory();
+        return reader;
+    }
+    PAIMON_RETURN_NOT_OK(stream->EnableCache().status());

Review Comment:
   This cast also succeeds for `LateMaterializingFileBatchReader` wrapping a 
reader that has no byte-range interface: the LM wrapper always derives from 
`PrefetchFileBatchReader`, while its `PreBufferRange()` simply returns an empty 
vector when `prefetch_inner_` is null. Consequently, enabling LM causes an 
otherwise plain reader, such as Avro, to open an unnecessary second stream 
here. I reproduced one successful open without LM versus two opens with LM; 
failing only the second open makes the LM path fail with `cache stream open 
failed`, although the underlying reader can read the file normally. Please base 
the decision on the underlying format reader's actual capability while 
retaining LM behavior. The existing plain-reader capability tests need the 
LM-wrapped variant as well, asserting that it builds once and opens only one 
stream.



##########
src/paimon/format/orc/orc_file_batch_reader.cpp:
##########
@@ -211,10 +222,20 @@ Status OrcFileBatchReader::SeekToRow(uint64_t row_number) 
{
 }
 
 Result<std::vector<std::pair<uint64_t, uint64_t>>> 
OrcFileBatchReader::PreBufferRange() {
-    return reader_->PreBufferRange(target_column_ids_);
+    using ByteRanges = std::vector<std::pair<uint64_t, uint64_t>>;
+    std::pair<uint64_t, uint64_t> row_range;
+    PAIMON_ASSIGN_OR_RAISE(ByteRanges ranges,
+                           reader_->PreBufferRange(target_column_ids_, 
&row_range));
+    pre_buffered_row_range_ = row_range;
+    return ranges;
 }
 
 Result<BatchReader::ReadBatch> OrcFileBatchReader::NextBatch() {
+    uint64_t next_row = reader_->GetNextRowToRead();
+    if (read_ahead_ && (!pre_buffered_row_range_ || next_row < 
pre_buffered_row_range_->first ||
+                        next_row >= pre_buffered_row_range_->second)) {
+        PAIMON_RETURN_NOT_OK(read_ahead_->PrepareRanges(this));

Review Comment:
   Preparing the next stripe only in `NextBatch()` is too late when reading 
involves a seek. The prefetch loop calls `EnsureReaderPosition()` first, and 
ORC's `seekToRow()` can initialize the target stripe, seek within a row group, 
and read payload before this code runs. In a real three-stripe ORC probe, after 
warming stripe 0, seeking into stripe 2 synchronously read all 7 bytes of that 
stripe's payload; the following `NextBatch()` then fetched the same 7 bytes 
again through the cache. Please prepare the target stripe from the requested 
row before the underlying seek performs I/O. A regression should route the 
format reads through `CacheInputStream` and assert physical reads/cache hits 
around the seek. `DiscoverStripeRangesOnDemand` currently passes a separate raw 
`CountingStream` directly to ORC, so its increasing cache-I/O counter does not 
demonstrate that decoding consumes the fetched bytes.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to