This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new 43f55f0010a branch-4.2: [perf](csv reader) Speed up CSV parsing 
(#70361)
43f55f0010a is described below

commit 43f55f0010af65bc5d45f486b4b07d10304b8372
Author: hui lai <[email protected]>
AuthorDate: Fri Nov 20 13:12:14 2026 +0800

    branch-4.2: [perf](csv reader) Speed up CSV parsing (#70361)
    
    Backports #69716 and #69875 to branch-4.2.
    
    Restore sequential prefetch for remote CSV reads in FileScannerV2 and
    reduce per-row and per-cell parsing overhead. The CSV string fast path
    is adapted to the parser APIs on branch-4.2, which lacks the master-only
    Hive OpenCSV parser.
    
    Local compilation and tests were skipped as requested.
---
 be/src/format_v2/delimited_text/csv_reader.cpp     |  45 ++-
 be/src/format_v2/delimited_text/csv_reader.h       |   1 +
 .../delimited_text/delimited_text_reader.cpp       | 103 +++++--
 .../delimited_text/delimited_text_reader.h         |  26 ++
 be/src/io/fs/buffered_reader.cpp                   |  43 ++-
 be/src/io/fs/buffered_reader.h                     |   8 +
 .../format_v2/delimited_text/csv_reader_test.cpp   | 335 ++++++++++++++++++++-
 .../format_v2/delimited_text/text_reader_test.cpp  |  32 +-
 be/test/io/fs/buffered_reader_test.cpp             | 181 +++++++++--
 9 files changed, 700 insertions(+), 74 deletions(-)

diff --git a/be/src/format_v2/delimited_text/csv_reader.cpp 
b/be/src/format_v2/delimited_text/csv_reader.cpp
index 95144d1cc95..0b0befc69b5 100644
--- a/be/src/format_v2/delimited_text/csv_reader.cpp
+++ b/be/src/format_v2/delimited_text/csv_reader.cpp
@@ -22,6 +22,7 @@
 
 #include "core/assert_cast.h"
 #include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
 #include "core/data_type/data_type_string.h"
 #include "core/data_type_serde/data_type_string_serde.h"
 #include "format/file_reader/new_plain_binary_line_reader.h"
@@ -199,6 +200,14 @@ void CsvReader::_split_line(const Slice& line) {
             append_value(value_start, separator_position - value_start);
             value_start = separator_position + _value_separator.size();
         }
+    } else if (_value_separator.size() == 1) {
+        const char separator = _value_separator[0];
+        for (size_t i = 0; i < line.size; ++i) {
+            if (line.data[i] == separator) {
+                append_value(value_start, i - value_start);
+                value_start = i + 1;
+            }
+        }
     } else {
         for (size_t i = 0; i < line.size;) {
             if (starts_with_at(line, i, _value_separator)) {
@@ -214,15 +223,29 @@ void CsvReader::_split_line(const Slice& line) {
     append_value(value_start, line.size - value_start);
 }
 
+void CsvReader::_prepare_columns(std::vector<MutableColumnPtr>& columns, 
size_t batch_size) {
+    for (const auto& column : _requested_columns) {
+        if (!column.nullable_string_fast_path) {
+            continue;
+        }
+        auto& nullable = 
assert_cast<ColumnNullable&>(*columns[column.block_position.value()]);
+        auto& strings = 
assert_cast<ColumnString&>(nullable.get_nested_column());
+        strings.get_offsets().reserve(strings.size() + batch_size);
+        nullable.get_null_map_data().reserve(nullable.size() + batch_size);
+    }
+}
+
 Status CsvReader::_deserialize_one_cell(const RequestedColumn& column, 
IColumn* output,
                                         Slice value) {
     DORIS_CHECK(output != nullptr);
     if (column.nullable_string_fast_path) {
-        auto& null_column = assert_cast<ColumnNullable&>(*output);
-        // String is the hottest CSV type. Avoid the generic nullable serde 
wrapper here:
-        // deserialize directly into the nested string column and append the 
null map bit ourselves.
+        // _prepare_columns validates the concrete types once per batch, after 
copy-on-write.
+        auto& null_column = assert_cast<ColumnNullable&, 
TypeCheckOnRelease::DISABLE>(*output);
+        auto& string_column = assert_cast<ColumnString&, 
TypeCheckOnRelease::DISABLE>(
+                null_column.get_nested_column());
         if (_empty_field_as_null && value.size == 0) {
-            null_column.insert_data(nullptr, 0);
+            string_column.insert_default();
+            null_column.get_null_map_data().push_back(1);
             return Status::OK();
         }
         // CSV keeps empty-field handling separate from null_format matching. 
An empty
@@ -231,16 +254,16 @@ Status CsvReader::_deserialize_one_cell(const 
RequestedColumn& column, IColumn*
         const bool quoted = _options.converted_from_string && 
value.trim_double_quotes();
         if (!quoted && _options.null_len > 0 && value.size == 
_options.null_len &&
             std::memcmp(value.data, _options.null_format, value.size) == 0) {
-            null_column.insert_data(nullptr, 0);
+            string_column.insert_default();
+            null_column.get_null_map_data().push_back(1);
             return Status::OK();
         }
-        static DataTypeStringSerDe string_serde(TYPE_STRING);
-        auto status = 
string_serde.deserialize_one_cell_from_csv(null_column.get_nested_column(),
-                                                                 value, 
_options);
-        if (!status.ok()) {
-            null_column.insert_data(nullptr, 0);
-            return Status::OK();
+        // Keep the same unescaping as StringSerDe, which always succeeds for 
CSV strings.
+        if (_options.escape_char != 0) {
+            escape_string_for_csv(value.data, &value.size, 
_options.escape_char,
+                                  _options.quote_char);
         }
+        string_column.insert_data(value.data, value.size);
         null_column.get_null_map_data().push_back(0);
         return Status::OK();
     }
diff --git a/be/src/format_v2/delimited_text/csv_reader.h 
b/be/src/format_v2/delimited_text/csv_reader.h
index dd1d77d5a34..ce76aed883d 100644
--- a/be/src/format_v2/delimited_text/csv_reader.h
+++ b/be/src/format_v2/delimited_text/csv_reader.h
@@ -58,6 +58,7 @@ private:
     Status _create_line_reader() override;
     Status _validate_line(const Slice& line) override;
     void _split_line(const Slice& line) override;
+    void _prepare_columns(std::vector<MutableColumnPtr>& columns, size_t 
batch_size) override;
     Status _deserialize_one_cell(const RequestedColumn& column, IColumn* 
output,
                                  Slice value) override;
     Slice _normalize_value(Slice value) const override;
diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp 
b/be/src/format_v2/delimited_text/delimited_text_reader.cpp
index 6600a2f1028..71f64ed2c49 100644
--- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp
+++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp
@@ -37,11 +37,13 @@
 #include "format_v2/column_mapper.h"
 #include "format_v2/materialized_reader_util.h"
 #include "io/file_factory.h"
+#include "io/fs/buffered_reader.h"
 #include "io/fs/tracing_file_reader.h"
 #include "runtime/descriptors.h"
 #include "runtime/file_scan_profile.h"
 #include "runtime/runtime_state.h"
 #include "util/decompressor.h"
+#include "util/defer_op.h"
 #include "util/string_util.h"
 #include "util/utf8_check.h"
 
@@ -191,11 +193,21 @@ void DelimitedTextReader::_init_profile() {
     _text_profile.create_line_reader_time =
             ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CreateLineReaderTime", 
DELIMITED_TEXT_PROFILE, 1);
     _text_profile.read_line_time =
-            ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadLineTime", 
DELIMITED_TEXT_PROFILE, 1);
+            ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SampledReadLineTime", 
DELIMITED_TEXT_PROFILE, 1);
     _text_profile.split_line_time =
-            ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SplitLineTime", 
DELIMITED_TEXT_PROFILE, 1);
-    _text_profile.deserialize_time =
-            ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DeserializeTime", 
DELIMITED_TEXT_PROFILE, 1);
+            ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SampledSplitLineTime", 
DELIMITED_TEXT_PROFILE, 1);
+    _text_profile.deserialize_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, 
"SampledDeserializeTime",
+                                                                
DELIMITED_TEXT_PROFILE, 1);
+    _text_profile.split_prefix_time =
+            ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SplitPrefixTime", 
DELIMITED_TEXT_PROFILE, 1);
+    _text_profile.sampled_read_line_calls = ADD_CHILD_COUNTER_WITH_LEVEL(
+            _profile, "SampledReadLineCalls", TUnit::UNIT, 
DELIMITED_TEXT_PROFILE, 1);
+    _text_profile.sampled_parsed_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
+            _profile, "SampledParsedRows", TUnit::UNIT, 
DELIMITED_TEXT_PROFILE, 1);
+    _profile->add_info_string(
+            "DelimitedTextTiming",
+            "Stage timers are unscaled samples of every 1024th read_line call "
+            "(starting with the first); DelimitedTextReader is full elapsed 
time");
     _text_profile.conjunct_filter_time =
             ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ConjunctFilterTime", 
DELIMITED_TEXT_PROFILE, 1);
     _text_profile.delete_conjunct_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(
@@ -284,6 +296,7 @@ Status DelimitedTextReader::init(RuntimeState* state) {
 
 Status DelimitedTextReader::infer_schema(RuntimeState* state,
                                          std::vector<ColumnDefinition>* 
schema) {
+    Defer profile_flush {[this] { _flush_profile(); }};
     _schema_only = true;
     RETURN_IF_ERROR(init(state));
     if (_file_description->range_start_offset != 0) {
@@ -359,8 +372,13 @@ Status 
DelimitedTextReader::open(std::shared_ptr<FileScanRequest> request) {
     return Status::OK();
 }
 
+void DelimitedTextReader::set_batch_size(size_t batch_size) {
+    _batch_size = std::max(size_t {1}, batch_size);
+}
+
 Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* 
eof) {
     SCOPED_TIMER(_text_profile.total_time);
+    Defer profile_flush {[this] { _flush_profile(); }};
     DORIS_CHECK(file_block != nullptr);
     DORIS_CHECK(rows != nullptr);
     DORIS_CHECK(eof != nullptr);
@@ -368,7 +386,9 @@ Status DelimitedTextReader::get_block(Block* file_block, 
size_t* rows, bool* eof
         return Status::InternalError("{} v2 reader is not open", _reader_name);
     }
 
-    const auto batch_size = _runtime_state != nullptr ? 
_runtime_state->batch_size() : 4096;
+    const size_t batch_size =
+            _batch_size != 0 ? _batch_size
+                             : (_runtime_state != nullptr ? 
_runtime_state->batch_size() : 4096);
     const auto max_block_bytes = _runtime_state != nullptr
                                          ? 
_runtime_state->preferred_block_size_bytes()
                                          : std::numeric_limits<size_t>::max();
@@ -378,6 +398,7 @@ Status DelimitedTextReader::get_block(Block* file_block, 
size_t* rows, bool* eof
     {
         auto columns_guard = file_block->mutate_columns_scoped();
         auto& columns = columns_guard.mutable_columns();
+        _prepare_columns(columns, batch_size);
         // Delimited text readers are column-pruned but not lazy materialized: 
all file-local
         // columns requested by TableReader are decoded before file-local 
conjuncts are evaluated.
         while (*rows < batch_size && !_line_reader_eof &&
@@ -436,6 +457,7 @@ Status DelimitedTextReader::get_block(Block* file_block, 
size_t* rows, bool* eof
 Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& 
request,
                                                  FileAggregateResult* result) {
     SCOPED_TIMER(_text_profile.total_time);
+    Defer profile_flush {[this] { _flush_profile(); }};
     DORIS_CHECK(result != nullptr);
     if (request.agg_type != TPushAggOp::type::COUNT) {
         return Status::NotSupported("{} v2 reader only supports COUNT 
aggregate pushdown",
@@ -460,7 +482,7 @@ Status DelimitedTextReader::get_aggregate_result(const 
FileAggregateRequest& req
             break;
         }
         if (line.size == 0) {
-            update_counter(_text_profile.empty_lines_read, 1);
+            ++_pending_profile.empty_lines_read;
             if (_empty_line_as_record() ||
                 (_runtime_state != nullptr && 
_runtime_state->is_read_csv_empty_line_as_null())) {
                 ++count;
@@ -486,6 +508,13 @@ Status DelimitedTextReader::close() {
         _line_reader.reset();
     }
     _decompressor.reset();
+    // Stream pipes are shared with the load context; preserve their existing 
ownership and
+    // cancellation behavior. Only filesystem readers enter the sequential 
prefetch path.
+    if (_file_reader != nullptr && _scan_params->file_type != 
TFileType::FILE_STREAM) {
+        // Drain prefetch workers before publishing their counters or 
releasing IOContext refs.
+        RETURN_IF_ERROR(_file_reader->close());
+        _file_reader->collect_profile_before_close();
+    }
     _file_reader.reset();
     _tracing_file_reader.reset();
     _requested_columns.clear();
@@ -605,8 +634,14 @@ Status DelimitedTextReader::_open_file() {
     } else {
         auto reader_options = 
FileFactory::get_reader_options(_runtime_state->query_options(),
                                                               
*_file_description);
-        auto file_reader = DORIS_TRY(FileFactory::create_file_reader(
-                *_system_properties, *_file_description, reader_options, 
_profile));
+        // Include the prefix used to align non-first splits. Unknown-size 
first splits read to
+        // EOF; PrefetchBufferedReader bounds this range by the opened file's 
physical size.
+        const auto prefetch_end = _size < 0 ? 
std::numeric_limits<size_t>::max()
+                                            : cast_set<size_t>(_start_offset + 
_size);
+        auto file_reader = DORIS_TRY(io::DelegateReader::create_file_reader(
+                _profile, *_system_properties, *_file_description, 
reader_options,
+                io::DelegateReader::AccessMode::SEQUENTIAL, _io_ctx,
+                io::PrefetchRange(_start_offset, prefetch_end)));
         _file_reader = _io_ctx && _io_ctx->file_reader_stats
                                ? 
std::make_shared<io::TracingFileReader>(std::move(file_reader),
                                                                          
_io_ctx->file_reader_stats)
@@ -624,7 +659,7 @@ Status DelimitedTextReader::_read_next_line(Slice* line, 
bool* eof) {
     DORIS_CHECK(line != nullptr);
     DORIS_CHECK(eof != nullptr);
     if (_align_split_prefix) {
-        SCOPED_TIMER(_text_profile.read_line_time);
+        SCOPED_TIMER(_text_profile.split_prefix_time);
         DCHECK_EQ(_skip_lines, 1);
         size_t skipped_lines = 0;
         auto* text_reader = 
assert_cast<NewPlainTextLineReader*>(_line_reader.get());
@@ -634,13 +669,19 @@ Status DelimitedTextReader::_read_next_line(Slice* line, 
bool* eof) {
         _align_split_prefix = false;
         _skip_lines = 0;
         _bom_removed = true;
-        update_counter(_text_profile.skipped_lines, skipped_lines);
+        _pending_profile.skipped_lines += skipped_lines;
     }
     while (true) {
         const uint8_t* ptr = nullptr;
         size_t size = 0;
         {
-            SCOPED_TIMER(_text_profile.read_line_time);
+            _sample_current_line =
+                    _profile != nullptr && (_read_line_calls++ % 
PROFILE_SAMPLE_INTERVAL == 0);
+            std::optional<ScopedRawTimer<MonotonicStopWatch, int64_t>> timer;
+            if (_sample_current_line) {
+                ++_pending_profile.sampled_read_line_calls;
+                timer.emplace(&_pending_profile.read_line_time);
+            }
             RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, 
&_line_reader_eof, _io_ctx.get()));
         }
         if (_line_reader_eof && size == 0) {
@@ -656,12 +697,12 @@ Status DelimitedTextReader::_read_next_line(Slice* line, 
bool* eof) {
         if (_skip_lines > 0) {
             --_skip_lines;
             _bom_removed = true;
-            update_counter(_text_profile.skipped_lines, 1);
+            ++_pending_profile.skipped_lines;
             continue;
         }
         *line = Slice(ptr, size);
         *eof = false;
-        update_counter(_text_profile.raw_lines_read, 1);
+        ++_pending_profile.raw_lines_read;
         return Status::OK();
     }
 }
@@ -671,12 +712,12 @@ Status DelimitedTextReader::_fill_columns_from_line(const 
Slice& line,
                                                     size_t* rows) {
     DORIS_CHECK(columns != nullptr);
     if (line.size == 0) {
-        update_counter(_text_profile.empty_lines_read, 1);
+        ++_pending_profile.empty_lines_read;
         if (!_empty_line_as_record()) {
             if (_runtime_state != nullptr && 
_runtime_state->is_read_csv_empty_line_as_null()) {
                 for (const auto& column : _requested_columns) {
                     
RETURN_IF_ERROR(_append_null((*columns)[column.block_position.value()].get()));
-                    update_counter(_text_profile.cells_deserialized, 1);
+                    ++_pending_profile.cells_deserialized;
                 }
                 ++(*rows);
             }
@@ -686,10 +727,17 @@ Status DelimitedTextReader::_fill_columns_from_line(const 
Slice& line,
     RETURN_IF_ERROR(_validate_line(line));
 
     {
-        SCOPED_TIMER(_text_profile.split_line_time);
+        std::optional<ScopedRawTimer<MonotonicStopWatch, int64_t>> timer;
+        if (_sample_current_line) {
+            ++_pending_profile.sampled_parsed_rows;
+            timer.emplace(&_pending_profile.split_line_time);
+        }
         _split_line(line);
     }
-    SCOPED_TIMER(_text_profile.deserialize_time);
+    std::optional<ScopedRawTimer<MonotonicStopWatch, int64_t>> timer;
+    if (_sample_current_line) {
+        timer.emplace(&_pending_profile.deserialize_time);
+    }
     if (_request->reject_invalid_rows) {
         const auto expected = 
cast_set<size_t>(_scan_params->num_of_columns_from_file);
         const bool ignore_extra = 
_scan_params->file_attributes.ignore_csv_redundant_col;
@@ -708,12 +756,31 @@ Status DelimitedTextReader::_fill_columns_from_line(const 
Slice& line,
                               ? _split_values[field_index]
                               : Slice(_options.null_format, _options.null_len);
         RETURN_IF_ERROR(_deserialize_one_cell(column, output, 
_normalize_value(value)));
-        update_counter(_text_profile.cells_deserialized, 1);
+        ++_pending_profile.cells_deserialized;
     }
     ++(*rows);
     return Status::OK();
 }
 
+void DelimitedTextReader::_flush_profile() {
+    update_counter(_text_profile.raw_lines_read, 
_pending_profile.raw_lines_read);
+    update_counter(_text_profile.empty_lines_read, 
_pending_profile.empty_lines_read);
+    update_counter(_text_profile.skipped_lines, 
_pending_profile.skipped_lines);
+    update_counter(_text_profile.cells_deserialized, 
_pending_profile.cells_deserialized);
+    update_counter(_text_profile.read_line_time, 
_pending_profile.read_line_time);
+    update_counter(_text_profile.split_line_time, 
_pending_profile.split_line_time);
+    update_counter(_text_profile.deserialize_time, 
_pending_profile.deserialize_time);
+    update_counter(_text_profile.sampled_read_line_calls, 
_pending_profile.sampled_read_line_calls);
+    update_counter(_text_profile.sampled_parsed_rows, 
_pending_profile.sampled_parsed_rows);
+    _pending_profile = {};
+}
+
+void DelimitedTextReader::_prepare_columns(std::vector<MutableColumnPtr>& 
columns,
+                                           size_t batch_size) {
+    (void)columns;
+    (void)batch_size;
+}
+
 Status DelimitedTextReader::_validate_line(const Slice& line) {
     (void)line;
     return Status::OK();
diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h 
b/be/src/format_v2/delimited_text/delimited_text_reader.h
index 645ba26eecd..4b7fc1f4014 100644
--- a/be/src/format_v2/delimited_text/delimited_text_reader.h
+++ b/be/src/format_v2/delimited_text/delimited_text_reader.h
@@ -55,6 +55,7 @@ public:
             TableColumnMapperOptions options) const override;
     Status open(std::shared_ptr<FileScanRequest> request) override;
     Status get_block(Block* file_block, size_t* rows, bool* eof) override;
+    void set_batch_size(size_t batch_size) override;
     Status get_aggregate_result(const FileAggregateRequest& request,
                                 FileAggregateResult* result) override;
     Status close() override;
@@ -64,6 +65,9 @@ protected:
         RuntimeProfile::Counter* total_time = nullptr;
         RuntimeProfile::Counter* open_file_time = nullptr;
         RuntimeProfile::Counter* create_line_reader_time = nullptr;
+        RuntimeProfile::Counter* split_prefix_time = nullptr;
+        RuntimeProfile::Counter* sampled_read_line_calls = nullptr;
+        RuntimeProfile::Counter* sampled_parsed_rows = nullptr;
         RuntimeProfile::Counter* read_line_time = nullptr;
         RuntimeProfile::Counter* split_line_time = nullptr;
         RuntimeProfile::Counter* deserialize_time = nullptr;
@@ -112,6 +116,8 @@ protected:
     // Split one logical line into `_split_values`. The common materialization 
path then resolves
     // requested field ordinals against `_split_values`.
     virtual void _split_line(const Slice& line) = 0;
+    // Prepare mutable output columns once per batch, after Block has resolved 
copy-on-write.
+    virtual void _prepare_columns(std::vector<MutableColumnPtr>& columns, 
size_t batch_size);
     // Deserialize a single normalized field into the requested output column 
using the
     // format-specific serde API.
     virtual Status _deserialize_one_cell(const RequestedColumn& column, 
IColumn* output,
@@ -178,6 +184,26 @@ private:
     Status _fill_columns_from_line(const Slice& line, 
std::vector<MutableColumnPtr>* columns,
                                    size_t* rows);
 
+    void _flush_profile();
+
+    // Publish exact counts once per API call instead of updating atomic 
counters per cell.
+    struct PendingProfile {
+        int64_t raw_lines_read = 0;
+        int64_t empty_lines_read = 0;
+        int64_t skipped_lines = 0;
+        int64_t cells_deserialized = 0;
+        int64_t read_line_time = 0;
+        int64_t split_line_time = 0;
+        int64_t deserialize_time = 0;
+        int64_t sampled_read_line_calls = 0;
+        int64_t sampled_parsed_rows = 0;
+    } _pending_profile;
+    // Keep sampling periodic across batches, including skipped headers and 
EOF probes.
+    static constexpr size_t PROFILE_SAMPLE_INTERVAL = 1024;
+    uint64_t _read_line_calls = 0;
+    bool _sample_current_line = false;
+    // Zero means use the runtime default until TableReader supplies its 
predicted row cap.
+    size_t _batch_size = 0;
     std::string _reader_name;
 };
 
diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
index 91eafdba431..d63f63bc286 100644
--- a/be/src/io/fs/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -413,6 +413,13 @@ void PrefetchBuffer::reset_offset(size_t offset) {
             _prefetched.notify_all();
             return;
         }
+        _merge_file_cache_stats();
+        // Snapshot on the consumer thread, not concurrently with scanner 
counter updates.
+        // Cached readers get per-buffer counters; uncached readers keep a 
null stats pointer.
+        _prefetch_io_ctx = *_io_ctx;
+        if (_io_ctx->file_cache_stats != nullptr) {
+            _prefetch_io_ctx.file_cache_stats = &_file_cache_stats;
+        }
         _buffer_status = BufferStatus::RESET;
         _offset = offset;
         _prefetched.notify_all();
@@ -470,7 +477,7 @@ void PrefetchBuffer::prefetch_buffer() {
 
     {
         SCOPED_RAW_TIMER(&_statis.read_time);
-        s = _reader->read_at(_offset, Slice {_buf.data(), buf_size}, &_len, 
_io_ctx);
+        s = _reader->read_at(_offset, Slice {_buf.data(), buf_size}, &_len, 
&_prefetch_io_ctx);
     }
     if (UNLIKELY(s.ok() && buf_size != _len)) {
         // This indicates that the data size returned by S3 object storage is 
smaller than what we requested,
@@ -497,6 +504,7 @@ void PrefetchBuffer::prefetch_buffer() {
                     _offset, s.to_string());
         _prefetch_status = std::move(s);
     }
+    _has_unmerged_cache_stats = _prefetch_io_ctx.file_cache_stats != nullptr;
     _buffer_status = BufferStatus::PREFETCHED;
     _prefetched.notify_all();
     // eof would come up with len == 0, it would be handled by read_buffer
@@ -582,6 +590,7 @@ Status PrefetchBuffer::read_buffer(size_t off, const char* 
out, size_t buf_len,
         if (UNLIKELY(BufferStatus::CLOSED == _buffer_status)) {
             return Status::OK();
         }
+        _merge_file_cache_stats();
     }
     RETURN_IF_ERROR(_prefetch_status);
     // there is only parquet would do not sequence read
@@ -612,15 +621,23 @@ Status PrefetchBuffer::read_buffer(size_t off, const 
char* out, size_t buf_len,
     return Status::OK();
 }
 
+void PrefetchBuffer::_merge_file_cache_stats() {
+    // Called only by the consumer with _lock held and no pending read. A 
buffer may be
+    // consumed in several slices; publish each completed request exactly once.
+    if (_has_unmerged_cache_stats) {
+        _io_ctx->file_cache_stats->merge_from(_file_cache_stats);
+        _file_cache_stats = {};
+        _has_unmerged_cache_stats = false;
+    }
+}
+
 void PrefetchBuffer::close() {
     std::unique_lock lck {_lock};
-    // in case _reader still tries to write to the buf after we close the 
buffer
-    if (!_prefetched.wait_for(lck,
-                              
std::chrono::milliseconds(config::buffered_reader_read_timeout_ms),
-                              [this]() { return _buffer_status != 
BufferStatus::PENDING; })) {
-        _prefetch_status = Status::TimedOut("time out when close prefetch 
buffer");
-        return;
-    }
+    // A foreground read timeout does not cancel the underlying I/O. Wait for 
it to finish
+    // before releasing scanner-owned IOContext pointees, including cache 
statistics and
+    // the cache-write limiter. Queued tasks observe CLOSED and do not start 
another read.
+    _prefetched.wait(lck, [this]() { return _buffer_status != 
BufferStatus::PENDING; });
+    _merge_file_cache_stats();
     _buffer_status = BufferStatus::CLOSED;
     _prefetched.notify_all();
     // Explicitly release the backing buffer here, in the calling (query) 
thread which has a
@@ -647,14 +664,14 @@ 
PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File
     }
     _io_ctx = _io_ctx_holder.get();
     if (buffer_size == -1L) {
-        buffer_size = config::remote_storage_read_buffer_mb * 1024 * 1024;
+        buffer_size = int64_t {config::remote_storage_read_buffer_mb} * 1024 * 
1024;
     }
     _size = _reader->size();
-    _whole_pre_buffer_size = buffer_size;
+    // Use the allocated capacity as the ring modulus. For example, 6 MiB 
allocates
+    // one 4 MiB buffer, so offset 4 MiB must wrap to slot 0, not index a 
missing slot 1.
+    const int64_t buffer_num = std::max<int64_t>(1, buffer_size / 
s_max_pre_buffer_size);
+    _whole_pre_buffer_size = buffer_num * s_max_pre_buffer_size;
     _file_range.end_offset = std::min(_file_range.end_offset, _size);
-    int buffer_num = buffer_size > s_max_pre_buffer_size
-                             ? cast_set<int>(buffer_size) / 
cast_set<int>(s_max_pre_buffer_size)
-                             : 1;
     std::function<void(PrefetchBuffer&)> sync_buffer = nullptr;
     if (profile != nullptr) {
         const char* prefetch_buffered_reader = "PrefetchBufferedReader";
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index 04566bea095..7b7f5031692 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -471,6 +471,11 @@ struct PrefetchBuffer : 
std::enable_shared_from_this<PrefetchBuffer>, public Pro
     io::FileReaderSPtr _reader;
     std::shared_ptr<const IOContext> _io_ctx_holder;
     const IOContext* _io_ctx = nullptr;
+    // Only the worker writes these statistics. The consumer merges them after 
observing
+    // completion under _lock, before reusing or closing the buffer.
+    IOContext _prefetch_io_ctx;
+    FileCacheStatistics _file_cache_stats;
+    bool _has_unmerged_cache_stats = false;
     PODArray<char> _buf;
     BufferStatus _buffer_status {BufferStatus::RESET};
     std::mutex _lock;
@@ -517,6 +522,9 @@ struct PrefetchBuffer : 
std::enable_shared_from_this<PrefetchBuffer>, public Pro
     void _collect_profile_at_runtime() override {}
 
     void _collect_profile_before_close() override;
+
+private:
+    void _merge_file_cache_stats();
 };
 
 constexpr int64_t s_max_pre_buffer_size = 4 * 1024 * 1024; // 4MB
diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp 
b/be/test/format_v2/delimited_text/csv_reader_test.cpp
index 32d4255acee..6c9682ad151 100644
--- a/be/test/format_v2/delimited_text/csv_reader_test.cpp
+++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp
@@ -20,9 +20,12 @@
 #include <gtest/gtest.h>
 
 #include <algorithm>
+#include <atomic>
+#include <cstring>
 #include <filesystem>
 #include <fstream>
 #include <memory>
+#include <thread>
 
 #include "common/config.h"
 #include "common/consts.h"
@@ -44,17 +47,95 @@
 #include "exprs/vexpr_context.h"
 #include "format/csv/csv_reader.h"
 #include "format_v2/column_mapper.h"
+#include "io/fs/stream_load_pipe.h"
 #include "io/io_common.h"
+#include "runtime/exec_env.h"
 #include "runtime/runtime_profile.h"
 #include "testutil/desc_tbl_builder.h"
 #include "testutil/mock/mock_runtime_state.h"
+#include "testutil/mock/obj_storage_client_test_stub.h"
 #include "testutil/scoped_temp_dir.h"
 #include "util/debug_points.h"
 #include "util/defer_op.h"
+#include "util/s3_util.h"
+#include "util/threadpool.h"
 
 namespace doris::format::csv {
 namespace {
 
+class CsvPrefetchS3Client : public io::ObjStorageClientTestStub {
+public:
+    const std::string data = 
"id,name,score\n1,alice,10\n2,bob,20\n3,carol,30\n";
+    const std::thread::id scanner_thread = std::this_thread::get_id();
+    std::atomic<size_t> foreground_reads {0};
+    std::atomic<size_t> reads {0};
+
+    io::ObjStorageHeadResult head_object(const io::ObjStoragePath&) override {
+        return {.resp = io::ObjStorageResponse::OK(),
+                .file_size = static_cast<long long>(data.size())};
+    }
+
+    io::ObjStorageResponse get_object(const io::ObjStoragePath&, void* buffer, 
size_t offset,
+                                      size_t bytes_read, size_t* size_return) 
override {
+        ++reads;
+        if (std::this_thread::get_id() == scanner_thread) {
+            ++foreground_reads;
+        }
+        EXPECT_LE(offset + bytes_read, data.size());
+        *size_return = std::min(bytes_read, data.size() - offset);
+        std::memcpy(buffer, data.data() + offset, *size_return);
+        return io::ObjStorageResponse::OK();
+    }
+
+    // Only HEAD and GET belong to this read test. Explicit overrides avoid 
generating unused
+    // gMock expectation helpers for this anonymous-namespace client.
+    io::ObjStorageUploadResult create_multipart_upload(const 
io::ObjStoragePath&) override {
+        return {.resp = unexpected_operation()};
+    }
+
+    io::ObjStorageResponse put_object(const io::ObjStoragePath&, 
std::string_view) override {
+        return unexpected_operation();
+    }
+
+    io::ObjStorageUploadResult upload_part(const io::ObjStoragePath&, const 
std::string&,
+                                           std::string_view, int) override {
+        return {.resp = unexpected_operation()};
+    }
+
+    io::ObjStorageResponse complete_multipart_upload(
+            const io::ObjStoragePath&, const std::string&,
+            const std::vector<io::ObjStorageCompletedPart>&) override {
+        return unexpected_operation();
+    }
+
+    io::ObjStorageResponse delete_objects(const io::ObjStoragePath&,
+                                          std::vector<std::string>) override {
+        return unexpected_operation();
+    }
+
+    io::ObjStorageResponse delete_object(const io::ObjStoragePath&) override {
+        return unexpected_operation();
+    }
+
+    std::string generate_presigned_url(const io::ObjStoragePath&, int64_t) 
override {
+        ADD_FAILURE() << "Unexpected signed URL request in CSV prefetch test";
+        return {};
+    }
+
+    io::ObjStorageListPageResult list_objects_page(const io::ObjStoragePath&,
+                                                   std::string_view) override {
+        return {.resp = unexpected_operation()};
+    }
+
+private:
+    static io::ObjStorageResponse unexpected_operation() {
+        ADD_FAILURE() << "Unexpected object storage operation in CSV prefetch 
test";
+        return {.status = {TStatusCode::NOT_IMPLEMENTED_ERROR,
+                           "CSV prefetch test only supports HEAD and GET"},
+                .http_code = 0};
+    }
+};
+
 TFileScanRangeParams csv_scan_params() {
     TFileScanRangeParams params;
     params.__set_format_type(TFileFormatType::FORMAT_CSV_PLAIN);
@@ -512,6 +593,78 @@ protected:
     TFileScanRangeParams _params;
 };
 
+TEST_F(CsvV2ReaderTest, S3PrefetchPreservesSplitRowsAndReadStatistics) {
+    auto* env = ExecEnv::GetInstance();
+    auto saved_pool = std::move(env->_buffered_reader_prefetch_thread_pool);
+    const auto saved_in_memory_size = config::in_memory_file_size;
+    Defer restore([&] {
+        env->_buffered_reader_prefetch_thread_pool.reset();
+        env->_buffered_reader_prefetch_thread_pool = std::move(saved_pool);
+        config::in_memory_file_size = saved_in_memory_size;
+        S3ClientFactory::instance().clear_client_creator_for_test();
+    });
+    ASSERT_TRUE(ThreadPoolBuilder("CsvPrefetchTest")
+                        .set_min_threads(1)
+                        .set_max_threads(1)
+                        .build(&env->_buffered_reader_prefetch_thread_pool)
+                        .ok());
+    // Keep the small fixture on the same prefetch path as a large S3 CSV file.
+    config::in_memory_file_size = 0;
+    auto client = std::make_shared<CsvPrefetchS3Client>();
+    S3ClientFactory::instance().set_client_creator_for_test(
+            [client](const S3ClientConf&) { return client; });
+    _params.__set_file_type(TFileType::FILE_S3);
+    _state._query_options.__set_enable_file_cache(false);
+
+    // Cover a complete file, a split starting inside Alice's row, and an 
unknown-size first split.
+    for (const int64_t split_start : {0, 17, -1}) {
+        SCOPED_TRACE(split_start);
+        auto properties = std::make_shared<io::FileSystemProperties>();
+        properties->system_type = TFileType::FILE_S3;
+        properties->properties = {{"AWS_ENDPOINT", 
"http://csv-prefetch.invalid"},
+                                  {"AWS_REGION", "test-region"}};
+        auto desc = std::make_unique<io::FileDescription>();
+        desc->path = "s3://csv-prefetch/reader.csv";
+        desc->range_start_offset = std::max<int64_t>(0, split_start);
+        desc->file_size = split_start < 0 ? -1 : 
static_cast<int64_t>(client->data.size());
+        desc->range_size = desc->file_size - desc->range_start_offset;
+        io::FileReaderStats stats;
+        auto io_ctx = std::make_shared<io::IOContext>();
+        io_ctx->file_reader_stats = &stats;
+        auto reader =
+                std::make_unique<CsvReader>(properties, desc, io_ctx, 
&_profile, &_params, _slots);
+        ASSERT_TRUE(reader->init(&_state).ok());
+        std::vector<ColumnDefinition> schema;
+        ASSERT_TRUE(reader->get_schema(&schema).ok());
+        auto request = std::make_shared<FileScanRequest>();
+        for (int32_t id = 0; id < 3; ++id) {
+            request->non_predicate_columns.push_back(
+                    LocalColumnIndex::top_level(LocalColumnId(id)));
+            request->local_positions.emplace(LocalColumnId(id), 
LocalIndex(id));
+        }
+        ASSERT_TRUE(reader->open(request).ok());
+        auto block = make_block(schema, {0, 1, 2});
+        size_t rows = 0;
+        bool eof = false;
+        ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+        ASSERT_EQ(rows, split_start > 0 ? 2 : 3);
+        EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 
split_start > 0 ? 2 : 1);
+        EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, rows - 
1), "carol");
+        EXPECT_GT(stats.read_calls, 0);
+        EXPECT_GT(stats.read_bytes, 0);
+        const auto previous_prefetch_io = counter_value(&_profile, 
"PreRequestIO");
+        ASSERT_TRUE(reader->close().ok());
+        EXPECT_GT(counter_value(&_profile, "PreRequestIO"), 
previous_prefetch_io);
+        EXPECT_GT(counter_value(&_profile, "PreRequestBytes"), 0);
+        EXPECT_GT(counter_value(&_profile, "ReadTime"), 0);
+        const auto collected_prefetch_io = counter_value(&_profile, 
"PreRequestIO");
+        ASSERT_TRUE(reader->close().ok());
+        EXPECT_EQ(counter_value(&_profile, "PreRequestIO"), 
collected_prefetch_io);
+    }
+    EXPECT_GT(client->reads.load(), 0);
+    EXPECT_EQ(client->foreground_reads.load(), 0);
+}
+
 // Scenario: CSV v2 exposes FE-provided file slots as nullable file-local 
schema using column_idxs
 // as CSV field ordinals.
 TEST_F(CsvV2ReaderTest, SchemaUsesSlotTypesAndColumnIdxs) {
@@ -599,9 +752,9 @@ TEST_F(CsvV2ReaderTest, 
ProfileCountersTrackReadParseDeserializeAndFilter) {
 
     EXPECT_NE(_profile.get_counter("OpenFileTime"), nullptr);
     EXPECT_NE(_profile.get_counter("CreateLineReaderTime"), nullptr);
-    EXPECT_NE(_profile.get_counter("ReadLineTime"), nullptr);
-    EXPECT_NE(_profile.get_counter("SplitLineTime"), nullptr);
-    EXPECT_NE(_profile.get_counter("DeserializeTime"), nullptr);
+    EXPECT_NE(_profile.get_counter("SampledReadLineTime"), nullptr);
+    EXPECT_NE(_profile.get_counter("SampledSplitLineTime"), nullptr);
+    EXPECT_NE(_profile.get_counter("SampledDeserializeTime"), nullptr);
     EXPECT_NE(_profile.get_counter("ConjunctFilterTime"), nullptr);
     EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr);
     EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3);
@@ -616,6 +769,162 @@ TEST_F(CsvV2ReaderTest, 
ProfileCountersTrackReadParseDeserializeAndFilter) {
     EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6);
 }
 
+// A row cap can change between calls. It must override the runtime default 
without dropping
+// records, and a zero prediction must still make progress.
+TEST_F(CsvV2ReaderTest, HonorsChangingBatchSizeAndZeroPrediction) {
+    std::ofstream output(_file_path, std::ios::binary);
+    output << "id,name,score\n";
+    for (int i = 1; i <= 7; ++i) {
+        output << i << ",name,10\n";
+    }
+    output.close();
+    _state._batch_size = 2;
+    auto reader = create_reader(_file_path, &_params, _slots, &_state, 
&_profile);
+    reader->set_batch_size(1);
+    auto request = std::make_shared<FileScanRequest>();
+    request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
+    request->local_positions.emplace(LocalColumnId(1), LocalIndex(1));
+    ASSERT_TRUE(reader->open(request).ok());
+    std::vector<ColumnDefinition> schema;
+    ASSERT_TRUE(reader->get_schema(&schema).ok());
+    int next_id = 1;
+    const std::vector<size_t> caps {1, 3, 0, 2, 2};
+    const std::vector<size_t> expected {1, 3, 1, 2, 0};
+    for (size_t i = 0; i < caps.size(); ++i) {
+        reader->set_batch_size(caps[i]);
+        auto block = make_block(schema, {0, 1});
+        size_t rows = 0;
+        bool eof = false;
+        ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+        ASSERT_EQ(rows, expected[i]);
+        EXPECT_EQ(eof, rows == 0);
+        for (size_t row = 0; row < rows; ++row) {
+            EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, row), 
next_id++);
+            EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 
row), "name");
+        }
+        EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), (next_id - 1) 
* 2);
+        EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), next_id - 1);
+    }
+    EXPECT_EQ(next_id, 8);
+    EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1);
+}
+
+TEST_F(CsvV2ReaderTest, BatchPredictionStillHonorsByteBudget) {
+    const bool saved = config::enable_adaptive_batch_size;
+    Defer restore {[&] { config::enable_adaptive_batch_size = saved; }};
+    config::enable_adaptive_batch_size = true;
+    _state._query_options.__set_preferred_block_size_bytes(1);
+    auto reader = create_reader(_file_path, &_params, _slots, &_state, 
&_profile);
+    reader->set_batch_size(100);
+    auto request = std::make_shared<FileScanRequest>();
+    request->local_positions.emplace(LocalColumnId(1), LocalIndex(0));
+    ASSERT_TRUE(reader->open(request).ok());
+    std::vector<ColumnDefinition> schema;
+    ASSERT_TRUE(reader->get_schema(&schema).ok());
+    for (const std::string expected : {"alice", "bob"}) {
+        auto block = make_block(schema, {1});
+        size_t rows = 0;
+        bool eof = false;
+        ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+        ASSERT_EQ(rows, 1);
+        EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), 
expected);
+    }
+}
+
+// Sampling spans batch boundaries; exact work counts are published before 
every return.
+TEST_F(CsvV2ReaderTest, ProfileSamplingSpansBatchesWithExactCounts) {
+    std::ofstream output(_file_path, std::ios::binary);
+    for (int i = 0; i < 2050; ++i) {
+        output << "1,name,10\n";
+    }
+    output.close();
+    _params.file_attributes.__set_header_type("");
+    auto reader = create_reader(_file_path, &_params, _slots, &_state, 
&_profile);
+    reader->set_batch_size(300);
+    auto request = std::make_shared<FileScanRequest>();
+    request->local_positions.emplace(LocalColumnId(1), LocalIndex(0));
+    ASSERT_TRUE(reader->open(request).ok());
+    std::vector<ColumnDefinition> schema;
+    ASSERT_TRUE(reader->get_schema(&schema).ok());
+    size_t total = 0;
+    bool eof = false;
+    while (!eof) {
+        auto block = make_block(schema, {1});
+        size_t rows = 0;
+        ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+        total += rows;
+        EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), total);
+        EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), total);
+    }
+    EXPECT_EQ(total, 2050);
+    EXPECT_EQ(counter_value(&_profile, "SampledReadLineCalls"), 3);
+    EXPECT_EQ(counter_value(&_profile, "SampledParsedRows"), 3);
+}
+
+// The single-byte split and direct string insertion must preserve NULL, 
quoted literals,
+// escaping, trailing empty fields, missing fields and reordered projections 
across batches.
+TEST_F(CsvV2ReaderTest, 
StringFastPathPreservesValuesForSingleAndMultiByteSeparators) {
+    auto type = make_nullable(std::make_shared<DataTypeString>());
+    std::vector<SlotDescriptor*> slots;
+    for (int i = 0; i < 3; ++i) {
+        slots.push_back(make_test_slot(&_pool, i, i, type, "s" + 
std::to_string(i)));
+    }
+    for (const std::string separator : {"|", "||"}) {
+        for (bool empty_as_null : {false, true}) {
+            SCOPED_TRACE(testing::Message() << separator << ", empty_as_null=" 
<< empty_as_null);
+            std::ofstream output(_file_path, std::ios::binary);
+            output << "NULL" << separator << "\"NULL\"" << separator << "\n";
+            output << "left" << separator << "a\\b" << separator << "tail\n";
+            output << "only\n";
+            output.close();
+            auto params = _params;
+            params.file_attributes.__set_header_type("");
+            params.file_attributes.__set_trim_double_quotes(true);
+            
params.file_attributes.text_params.__set_column_separator(separator);
+            params.file_attributes.text_params.__set_null_format("NULL");
+            
params.file_attributes.text_params.__set_empty_field_as_null(empty_as_null);
+            params.file_attributes.text_params.__set_escape('\\');
+            auto reader = create_reader(_file_path, &params, slots, &_state, 
&_profile);
+            reader->set_batch_size(1);
+            auto request = std::make_shared<FileScanRequest>();
+            request->local_positions.emplace(LocalColumnId(2), LocalIndex(0));
+            request->local_positions.emplace(LocalColumnId(0), LocalIndex(1));
+            request->local_positions.emplace(LocalColumnId(1), LocalIndex(2));
+            ASSERT_TRUE(reader->open(request).ok());
+            std::vector<ColumnDefinition> schema;
+            ASSERT_TRUE(reader->get_schema(&schema).ok());
+            for (int row = 0; row < 3; ++row) {
+                auto block = make_block(schema, {2, 0, 1});
+                // Hold shared columns while the reader acquires its mutable 
batch.
+                auto shared = block.get_by_position(0).column;
+                size_t rows = 0;
+                bool eof = false;
+                ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+                ASSERT_EQ(rows, 1);
+                EXPECT_EQ(shared->size(), 0);
+                const auto& c0 = *block.get_by_position(0).column;
+                const auto& c1 = *block.get_by_position(1).column;
+                const auto& c2 = *block.get_by_position(2).column;
+                if (row == 0) {
+                    EXPECT_EQ(is_null_at(c0, 0), empty_as_null);
+                    EXPECT_EQ(nullable_string_at(c0, 0), "");
+                    EXPECT_TRUE(is_null_at(c1, 0));
+                    EXPECT_FALSE(is_null_at(c2, 0));
+                    EXPECT_EQ(nullable_string_at(c2, 0), "NULL");
+                } else if (row == 1) {
+                    EXPECT_EQ(nullable_string_at(c0, 0), "tail");
+                    EXPECT_EQ(nullable_string_at(c1, 0), "left");
+                    EXPECT_EQ(nullable_string_at(c2, 0), "ab");
+                } else {
+                    EXPECT_TRUE(is_null_at(c0, 0));
+                    EXPECT_EQ(nullable_string_at(c1, 0), "only");
+                    EXPECT_TRUE(is_null_at(c2, 0));
+                }
+            }
+        }
+    }
+}
+
 // Scenario: CSV has no embedded nested schema, but TableColumnMapper still 
needs semantic children
 // for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children 
from the slot type
 // while keeping the top-level local id as the CSV field ordinal from 
column_idxs.
@@ -675,6 +984,8 @@ TEST_F(CsvV2ReaderTest, 
LoadRejectsWrongFieldCountsWithoutShiftingFollowingRows)
     EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 1);
     EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 1), 4);
     EXPECT_EQ(io_ctx->load_filtered_rows, 2);
+    EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 4);
+    EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6);
 }
 
 // Scenario: CSV v2 honors FileScanRequest local positions, so TableReader can 
request a subset of
@@ -727,6 +1038,9 @@ TEST_F(CsvV2ReaderTest, 
InvalidUtf8FailsWhenValidationEnabled) {
     bool eof = false;
     const auto status = reader->get_block(&block, &rows, &eof);
     EXPECT_FALSE(status.ok());
+    EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 1);
+    EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1);
+    EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 0);
     EXPECT_TRUE(status.to_string().find("Only support csv data in utf8 codec") 
!= std::string::npos)
             << status;
 }
@@ -927,6 +1241,17 @@ TEST_F(CsvV2ReaderTest, 
StreamInputRequiresLoadIdBeforeOpeningPipe) {
             << status;
 }
 
+TEST_F(CsvV2ReaderTest, ClosePreservesSharedStreamPipe) {
+    _params.__set_file_type(TFileType::FILE_STREAM);
+    auto reader = create_unknown_size_reader(_file_path, &_params, _slots, 
&_state, &_profile);
+    auto pipe = std::make_shared<io::StreamLoadPipe>();
+    reader->_file_reader = pipe;
+    ASSERT_TRUE(reader->close().ok());
+    EXPECT_FALSE(pipe->closed());
+    reader.reset();
+    EXPECT_FALSE(pipe->closed());
+}
+
 // Scenario: CSV has no footer row count, so v2 COUNT pushdown scans the split 
and returns the
 // counted row count through FileAggregateResult.
 TEST_F(CsvV2ReaderTest, CountAggregateScansRows) {
@@ -939,6 +1264,10 @@ TEST_F(CsvV2ReaderTest, CountAggregateScansRows) {
     FileAggregateResult aggregate_result;
     ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, 
&aggregate_result).ok());
     EXPECT_EQ(aggregate_result.count, 2);
+    EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 2);
+    EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1);
+    EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 0);
+    EXPECT_EQ(counter_value(&_profile, "SampledParsedRows"), 0);
 }
 
 TEST_F(CsvV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) {
diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp 
b/be/test/format_v2/delimited_text/text_reader_test.cpp
index 4927ee7622e..2fe1aa84bef 100644
--- a/be/test/format_v2/delimited_text/text_reader_test.cpp
+++ b/be/test/format_v2/delimited_text/text_reader_test.cpp
@@ -408,9 +408,9 @@ TEST_F(TextV2ReaderTest, 
ProfileCountersTrackReadParseDeserializeAndFilter) {
 
     EXPECT_NE(_profile.get_counter("OpenFileTime"), nullptr);
     EXPECT_NE(_profile.get_counter("CreateLineReaderTime"), nullptr);
-    EXPECT_NE(_profile.get_counter("ReadLineTime"), nullptr);
-    EXPECT_NE(_profile.get_counter("SplitLineTime"), nullptr);
-    EXPECT_NE(_profile.get_counter("DeserializeTime"), nullptr);
+    EXPECT_NE(_profile.get_counter("SampledReadLineTime"), nullptr);
+    EXPECT_NE(_profile.get_counter("SampledSplitLineTime"), nullptr);
+    EXPECT_NE(_profile.get_counter("SampledDeserializeTime"), nullptr);
     EXPECT_NE(_profile.get_counter("ConjunctFilterTime"), nullptr);
     EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr);
     EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3);
@@ -424,6 +424,32 @@ TEST_F(TextV2ReaderTest, 
ProfileCountersTrackReadParseDeserializeAndFilter) {
     EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6);
 }
 
+TEST_F(TextV2ReaderTest, HonorsBatchSizeAcrossEmptyAndFilteredBlocks) {
+    auto reader = create_reader(_file_path, &_params, _slots, &_state, 
&_profile);
+    reader->set_batch_size(1);
+    auto request = std::make_shared<FileScanRequest>();
+    request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
+    request->conjuncts = {
+            prepared_conjunct(&_state, 
std::make_shared<NullableIntGreaterThanExpr>(0, 1))};
+    ASSERT_TRUE(reader->open(request).ok());
+    std::vector<ColumnDefinition> schema;
+    ASSERT_TRUE(reader->get_schema(&schema).ok());
+    for (int batch = 0; batch < 3; ++batch) {
+        auto block = make_block(schema, {0});
+        size_t rows = 0;
+        bool eof = false;
+        ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+        EXPECT_EQ(rows, batch == 1 ? 1 : 0);
+        EXPECT_EQ(eof, batch == 2);
+        if (rows == 1) {
+            EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2);
+        }
+    }
+    EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 2);
+    EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 2);
+    EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 1);
+}
+
 TEST_F(TextV2ReaderTest, 
FormatProfilesKeepDistinctCountersInBothInitializationOrders) {
     auto expect_format_children = [](RuntimeProfile* profile) {
         TRuntimeProfileTree tree;
diff --git a/be/test/io/fs/buffered_reader_test.cpp 
b/be/test/io/fs/buffered_reader_test.cpp
index 75ed5e351dc..b8f96b339fc 100644
--- a/be/test/io/fs/buffered_reader_test.cpp
+++ b/be/test/io/fs/buffered_reader_test.cpp
@@ -20,9 +20,12 @@
 #include <glog/logging.h>
 #include <gtest/gtest-message.h>
 #include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
 #include <limits.h>
 
+#include <algorithm>
 #include <atomic>
+#include <cstring>
 #include <memory>
 #include <ostream>
 #include <thread>
@@ -33,6 +36,7 @@
 #include "io/fs/local_file_system.h"
 #include "runtime/exec_env.h"
 #include "util/countdown_latch.h"
+#include "util/defer_op.h"
 #include "util/stopwatch.hpp"
 #include "util/threadpool.h"
 
@@ -150,7 +154,7 @@ public:
 
 protected:
     Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
-                        const io::IOContext* /*io_ctx*/) override {
+                        const io::IOContext* io_ctx) override {
         _read_started->count_down();
         _continue_read->wait();
         if (offset >= _size) {
@@ -158,7 +162,9 @@ protected:
             return Status::OK();
         }
         *bytes_read = std::min(_size - offset, result.size);
-        return Status::TimedOut("injected prefetch timeout");
+        memset(result.data, 0, *bytes_read);
+        io_ctx->file_cache_stats->bytes_read_from_remote += *bytes_read;
+        return Status::OK();
     }
 
 private:
@@ -352,37 +358,160 @@ TEST_F(BufferedReaderTest, test_miss) {
     EXPECT_EQ(45, bytes_read);
 }
 
-TEST_F(BufferedReaderTest, 
prefetch_task_keeps_reader_alive_after_close_timeout) {
+// A read timeout must not let close/destruction release the scanner's 
borrowed state.
+TEST_F(BufferedReaderTest, prefetch_close_drains_timed_out_reads) {
     const auto saved_timeout_ms = config::buffered_reader_read_timeout_ms;
-    config::buffered_reader_read_timeout_ms = 50;
-
-    CountDownLatch read_started(1);
-    CountDownLatch continue_read(1);
-    std::atomic<bool> reader_destroyed = false;
-    auto inner_reader = std::make_shared<BlockingFileReader>(1024, 
&read_started, &continue_read,
-                                                             
&reader_destroyed);
-    auto weak_reader = std::weak_ptr<io::FileReader>(inner_reader);
-    {
-        io::PrefetchBufferedReader reader(nullptr, std::move(inner_reader),
-                                          io::PrefetchRange(0, 1024));
+    Defer restore([&] { config::buffered_reader_read_timeout_ms = 
saved_timeout_ms; });
+    config::buffered_reader_read_timeout_ms = 20;
+
+    for (bool explicit_close : {false, true}) {
+        SCOPED_TRACE(explicit_close);
+        CountDownLatch read_started(1);
+        CountDownLatch continue_read(1);
+        CountDownLatch close_started(1);
+        CountDownLatch close_finished(1);
+        std::atomic<bool> reader_destroyed = false;
+        io::FileCacheStatistics stats;
+        auto io_ctx = std::make_shared<io::IOContext>();
+        io_ctx->file_cache_stats = &stats;
+        auto inner_reader = std::make_shared<BlockingFileReader>(1024, 
&read_started,
+                                                                 
&continue_read, &reader_destroyed);
+        auto reader = std::make_unique<io::PrefetchBufferedReader>(
+                nullptr, std::move(inner_reader), io::PrefetchRange(0, 1024), 
io_ctx);
         uint8_t buf[1];
         size_t bytes_read = 0;
-        Status st = reader.read_at(0, Slice {buf, 1}, &bytes_read);
-        EXPECT_TRUE(st.is<ErrorCode::TIMEOUT>());
-        EXPECT_TRUE(read_started.wait_for(std::chrono::seconds(1)));
-        EXPECT_FALSE(reader_destroyed.load());
+        EXPECT_TRUE(reader->read_at(0, Slice {buf, 1}, 
&bytes_read).is<ErrorCode::TIMEOUT>());
+        EXPECT_TRUE(read_started.wait_for(std::chrono::seconds(5)));
+        EXPECT_EQ(stats.bytes_read_from_remote, 0);
+
+        std::thread finish_read([&] {
+            close_started.wait();
+            // Hold the request beyond the old close timeout. Closing must 
remain blocked.
+            
EXPECT_FALSE(close_finished.wait_for(std::chrono::milliseconds(100)));
+            EXPECT_FALSE(reader_destroyed.load());
+            continue_read.count_down();
+        });
+        close_started.count_down();
+        if (explicit_close) {
+            EXPECT_TRUE(reader->close().ok());
+        }
+        reader.reset();
+        close_finished.count_down();
+        finish_read.join();
+        // Let the pool release its task captures before the 
destructor-observer goes out of scope.
+        ExecEnv::GetInstance()->buffered_reader_prefetch_thread_pool()->wait();
+        EXPECT_TRUE(reader_destroyed.load());
+        EXPECT_EQ(stats.bytes_read_from_remote, 1024);
     }
-    EXPECT_FALSE(reader_destroyed.load());
-    EXPECT_FALSE(weak_reader.expired());
+}
 
-    continue_read.count_down();
-    for (int i = 0; i < 100 && !reader_destroyed.load(); ++i) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+class PrefetchStatisticsFileReader : public MockOffsetFileReader {
+public:
+    PrefetchStatisticsFileReader(size_t size, io::FileCacheStatistics* 
scanner_stats,
+                                 CountDownLatch* started, CountDownLatch* 
resume)
+            : MockOffsetFileReader(size),
+              _scanner_stats(scanner_stats),
+              _started(started),
+              _resume(resume) {}
+
+    std::atomic<bool> used_scanner_stats = false;
+
+protected:
+    Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+                        const io::IOContext* io_ctx) override {
+        if (io_ctx->file_cache_stats == _scanner_stats) {
+            used_scanner_stats = true;
+        }
+        RETURN_IF_ERROR(MockOffsetFileReader::read_at_impl(offset, result, 
bytes_read, io_ctx));
+        auto* stats = io_ctx->file_cache_stats;
+        stats->bytes_read_from_remote += *bytes_read;
+        ++stats->num_remote_io_total;
+        stats->peer_hosts.insert("prefetch-test");
+        stats->remote_only_on_miss_triggered = true;
+        stats->remote_only_on_miss_threshold_bytes = 42;
+        _started->count_down();
+        _resume->wait();
+        return Status::OK();
     }
-    EXPECT_TRUE(reader_destroyed.load());
-    EXPECT_TRUE(weak_reader.expired());
 
-    config::buffered_reader_read_timeout_ms = saved_timeout_ms;
+private:
+    io::FileCacheStatistics* _scanner_stats;
+    CountDownLatch* _started;
+    CountDownLatch* _resume;
+};
+
+TEST_F(BufferedReaderTest, 
prefetch_merges_worker_cache_statistics_on_consumer) {
+    constexpr size_t buffer_size = 4 * 1024 * 1024;
+    constexpr size_t file_size = 5 * buffer_size;
+    CountDownLatch started(4);
+    CountDownLatch resume(1);
+    io::FileCacheStatistics stats;
+    auto io_ctx = std::make_shared<io::IOContext>();
+    io_ctx->file_cache_stats = &stats;
+    auto inner =
+            std::make_shared<PrefetchStatisticsFileReader>(file_size, &stats, 
&started, &resume);
+    io::PrefetchBufferedReader reader(nullptr, inner, io::PrefetchRange(0, 
file_size), io_ctx,
+                                      4 * buffer_size);
+    std::thread release_workers([&] {
+        EXPECT_TRUE(started.wait_for(std::chrono::seconds(5)));
+        // All four workers have updated their counters, but the consumer has 
not merged them.
+        EXPECT_FALSE(inner->used_scanner_stats.load());
+        EXPECT_EQ(stats.bytes_read_from_remote, 0);
+        resume.count_down();
+    });
+    uint8_t buf[2];
+    size_t bytes_read = 0;
+    auto status = reader.read_at(0, Slice {buf, 1}, &bytes_read);
+    release_workers.join();
+    ASSERT_TRUE(status.ok());
+    EXPECT_EQ(stats.bytes_read_from_remote, buffer_size);
+    ASSERT_TRUE(reader.read_at(1, Slice {buf, 1}, &bytes_read).ok());
+    EXPECT_EQ(stats.bytes_read_from_remote, buffer_size); // No double merge 
for another slice.
+    // Reuse slot 0 without consuming slots 1..3; close must still collect 
their statistics.
+    ASSERT_TRUE(reader.read_at(4 * buffer_size, Slice {buf, 1}, 
&bytes_read).ok());
+    EXPECT_EQ(stats.bytes_read_from_remote, 2 * buffer_size);
+    ASSERT_TRUE(reader.close().ok());
+    EXPECT_EQ(stats.bytes_read_from_remote, file_size);
+    EXPECT_EQ(stats.num_remote_io_total, 5);
+    EXPECT_EQ(stats.peer_hosts.count("prefetch-test"), 1);
+    EXPECT_TRUE(stats.remote_only_on_miss_triggered);
+    EXPECT_EQ(stats.remote_only_on_miss_threshold_bytes, 42);
+    ASSERT_TRUE(reader.close().ok());
+    EXPECT_EQ(stats.bytes_read_from_remote, file_size);
+}
+
+TEST_F(BufferedReaderTest, prefetch_normalizes_ring_capacity) {
+    constexpr size_t mb = 1024 * 1024;
+    constexpr size_t file_size = 10 * mb;
+    std::vector<uint8_t> expected(file_size);
+    for (size_t i = 0; i < file_size; ++i) {
+        expected[i] = i % UCHAR_MAX;
+    }
+    const auto saved_buffer_mb = config::remote_storage_read_buffer_mb;
+    Defer restore([&] { config::remote_storage_read_buffer_mb = 
saved_buffer_mb; });
+    // Includes less than one buffer, a non-multiple of 4 MiB, and a 
multi-slot ring.
+    for (int capacity_mb : {0, 1, 6, 10, 16}) {
+        config::remote_storage_read_buffer_mb = capacity_mb;
+        for (size_t start : {size_t {0}, 4 * mb + 7}) {
+            SCOPED_TRACE(capacity_mb);
+            SCOPED_TRACE(start);
+            io::PrefetchBufferedReader reader(nullptr,
+                                              
std::make_shared<MockOffsetFileReader>(file_size),
+                                              io::PrefetchRange(start, 
file_size));
+            std::vector<uint8_t> data(3 * mb);
+            for (size_t offset = start; offset < file_size;) {
+                size_t bytes_read = 0;
+                ASSERT_TRUE(
+                        reader.read_at(offset, Slice {data.data(), 
data.size()}, &bytes_read).ok());
+                ASSERT_EQ(bytes_read, std::min(data.size(), file_size - 
offset));
+                ASSERT_TRUE(std::equal(data.begin(), data.begin() + bytes_read,
+                                       expected.begin() + offset))
+                        << offset;
+                offset += bytes_read;
+            }
+            ASSERT_TRUE(reader.close().ok());
+        }
+    }
 }
 
 TEST_F(BufferedReaderTest, test_read_amplify) {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to