This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 43f55f0010a branch-4.2: [perf](csv reader) Speed up CSV parsing
(#70361)
43f55f0010a is described below
commit 43f55f0010af65bc5d45f486b4b07d10304b8372
Author: hui lai <[email protected]>
AuthorDate: Fri Nov 20 13:12:14 2026 +0800
branch-4.2: [perf](csv reader) Speed up CSV parsing (#70361)
Backports #69716 and #69875 to branch-4.2.
Restore sequential prefetch for remote CSV reads in FileScannerV2 and
reduce per-row and per-cell parsing overhead. The CSV string fast path
is adapted to the parser APIs on branch-4.2, which lacks the master-only
Hive OpenCSV parser.
Local compilation and tests were skipped as requested.
---
be/src/format_v2/delimited_text/csv_reader.cpp | 45 ++-
be/src/format_v2/delimited_text/csv_reader.h | 1 +
.../delimited_text/delimited_text_reader.cpp | 103 +++++--
.../delimited_text/delimited_text_reader.h | 26 ++
be/src/io/fs/buffered_reader.cpp | 43 ++-
be/src/io/fs/buffered_reader.h | 8 +
.../format_v2/delimited_text/csv_reader_test.cpp | 335 ++++++++++++++++++++-
.../format_v2/delimited_text/text_reader_test.cpp | 32 +-
be/test/io/fs/buffered_reader_test.cpp | 181 +++++++++--
9 files changed, 700 insertions(+), 74 deletions(-)
diff --git a/be/src/format_v2/delimited_text/csv_reader.cpp
b/be/src/format_v2/delimited_text/csv_reader.cpp
index 95144d1cc95..0b0befc69b5 100644
--- a/be/src/format_v2/delimited_text/csv_reader.cpp
+++ b/be/src/format_v2/delimited_text/csv_reader.cpp
@@ -22,6 +22,7 @@
#include "core/assert_cast.h"
#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type_serde/data_type_string_serde.h"
#include "format/file_reader/new_plain_binary_line_reader.h"
@@ -199,6 +200,14 @@ void CsvReader::_split_line(const Slice& line) {
append_value(value_start, separator_position - value_start);
value_start = separator_position + _value_separator.size();
}
+ } else if (_value_separator.size() == 1) {
+ const char separator = _value_separator[0];
+ for (size_t i = 0; i < line.size; ++i) {
+ if (line.data[i] == separator) {
+ append_value(value_start, i - value_start);
+ value_start = i + 1;
+ }
+ }
} else {
for (size_t i = 0; i < line.size;) {
if (starts_with_at(line, i, _value_separator)) {
@@ -214,15 +223,29 @@ void CsvReader::_split_line(const Slice& line) {
append_value(value_start, line.size - value_start);
}
+void CsvReader::_prepare_columns(std::vector<MutableColumnPtr>& columns,
size_t batch_size) {
+ for (const auto& column : _requested_columns) {
+ if (!column.nullable_string_fast_path) {
+ continue;
+ }
+ auto& nullable =
assert_cast<ColumnNullable&>(*columns[column.block_position.value()]);
+ auto& strings =
assert_cast<ColumnString&>(nullable.get_nested_column());
+ strings.get_offsets().reserve(strings.size() + batch_size);
+ nullable.get_null_map_data().reserve(nullable.size() + batch_size);
+ }
+}
+
Status CsvReader::_deserialize_one_cell(const RequestedColumn& column,
IColumn* output,
Slice value) {
DORIS_CHECK(output != nullptr);
if (column.nullable_string_fast_path) {
- auto& null_column = assert_cast<ColumnNullable&>(*output);
- // String is the hottest CSV type. Avoid the generic nullable serde
wrapper here:
- // deserialize directly into the nested string column and append the
null map bit ourselves.
+ // _prepare_columns validates the concrete types once per batch, after
copy-on-write.
+ auto& null_column = assert_cast<ColumnNullable&,
TypeCheckOnRelease::DISABLE>(*output);
+ auto& string_column = assert_cast<ColumnString&,
TypeCheckOnRelease::DISABLE>(
+ null_column.get_nested_column());
if (_empty_field_as_null && value.size == 0) {
- null_column.insert_data(nullptr, 0);
+ string_column.insert_default();
+ null_column.get_null_map_data().push_back(1);
return Status::OK();
}
// CSV keeps empty-field handling separate from null_format matching.
An empty
@@ -231,16 +254,16 @@ Status CsvReader::_deserialize_one_cell(const
RequestedColumn& column, IColumn*
const bool quoted = _options.converted_from_string &&
value.trim_double_quotes();
if (!quoted && _options.null_len > 0 && value.size ==
_options.null_len &&
std::memcmp(value.data, _options.null_format, value.size) == 0) {
- null_column.insert_data(nullptr, 0);
+ string_column.insert_default();
+ null_column.get_null_map_data().push_back(1);
return Status::OK();
}
- static DataTypeStringSerDe string_serde(TYPE_STRING);
- auto status =
string_serde.deserialize_one_cell_from_csv(null_column.get_nested_column(),
- value,
_options);
- if (!status.ok()) {
- null_column.insert_data(nullptr, 0);
- return Status::OK();
+ // Keep the same unescaping as StringSerDe, which always succeeds for
CSV strings.
+ if (_options.escape_char != 0) {
+ escape_string_for_csv(value.data, &value.size,
_options.escape_char,
+ _options.quote_char);
}
+ string_column.insert_data(value.data, value.size);
null_column.get_null_map_data().push_back(0);
return Status::OK();
}
diff --git a/be/src/format_v2/delimited_text/csv_reader.h
b/be/src/format_v2/delimited_text/csv_reader.h
index dd1d77d5a34..ce76aed883d 100644
--- a/be/src/format_v2/delimited_text/csv_reader.h
+++ b/be/src/format_v2/delimited_text/csv_reader.h
@@ -58,6 +58,7 @@ private:
Status _create_line_reader() override;
Status _validate_line(const Slice& line) override;
void _split_line(const Slice& line) override;
+ void _prepare_columns(std::vector<MutableColumnPtr>& columns, size_t
batch_size) override;
Status _deserialize_one_cell(const RequestedColumn& column, IColumn*
output,
Slice value) override;
Slice _normalize_value(Slice value) const override;
diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp
b/be/src/format_v2/delimited_text/delimited_text_reader.cpp
index 6600a2f1028..71f64ed2c49 100644
--- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp
+++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp
@@ -37,11 +37,13 @@
#include "format_v2/column_mapper.h"
#include "format_v2/materialized_reader_util.h"
#include "io/file_factory.h"
+#include "io/fs/buffered_reader.h"
#include "io/fs/tracing_file_reader.h"
#include "runtime/descriptors.h"
#include "runtime/file_scan_profile.h"
#include "runtime/runtime_state.h"
#include "util/decompressor.h"
+#include "util/defer_op.h"
#include "util/string_util.h"
#include "util/utf8_check.h"
@@ -191,11 +193,21 @@ void DelimitedTextReader::_init_profile() {
_text_profile.create_line_reader_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CreateLineReaderTime",
DELIMITED_TEXT_PROFILE, 1);
_text_profile.read_line_time =
- ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadLineTime",
DELIMITED_TEXT_PROFILE, 1);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SampledReadLineTime",
DELIMITED_TEXT_PROFILE, 1);
_text_profile.split_line_time =
- ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SplitLineTime",
DELIMITED_TEXT_PROFILE, 1);
- _text_profile.deserialize_time =
- ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DeserializeTime",
DELIMITED_TEXT_PROFILE, 1);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SampledSplitLineTime",
DELIMITED_TEXT_PROFILE, 1);
+ _text_profile.deserialize_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile,
"SampledDeserializeTime",
+
DELIMITED_TEXT_PROFILE, 1);
+ _text_profile.split_prefix_time =
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SplitPrefixTime",
DELIMITED_TEXT_PROFILE, 1);
+ _text_profile.sampled_read_line_calls = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "SampledReadLineCalls", TUnit::UNIT,
DELIMITED_TEXT_PROFILE, 1);
+ _text_profile.sampled_parsed_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "SampledParsedRows", TUnit::UNIT,
DELIMITED_TEXT_PROFILE, 1);
+ _profile->add_info_string(
+ "DelimitedTextTiming",
+ "Stage timers are unscaled samples of every 1024th read_line call "
+ "(starting with the first); DelimitedTextReader is full elapsed
time");
_text_profile.conjunct_filter_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ConjunctFilterTime",
DELIMITED_TEXT_PROFILE, 1);
_text_profile.delete_conjunct_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(
@@ -284,6 +296,7 @@ Status DelimitedTextReader::init(RuntimeState* state) {
Status DelimitedTextReader::infer_schema(RuntimeState* state,
std::vector<ColumnDefinition>*
schema) {
+ Defer profile_flush {[this] { _flush_profile(); }};
_schema_only = true;
RETURN_IF_ERROR(init(state));
if (_file_description->range_start_offset != 0) {
@@ -359,8 +372,13 @@ Status
DelimitedTextReader::open(std::shared_ptr<FileScanRequest> request) {
return Status::OK();
}
+void DelimitedTextReader::set_batch_size(size_t batch_size) {
+ _batch_size = std::max(size_t {1}, batch_size);
+}
+
Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool*
eof) {
SCOPED_TIMER(_text_profile.total_time);
+ Defer profile_flush {[this] { _flush_profile(); }};
DORIS_CHECK(file_block != nullptr);
DORIS_CHECK(rows != nullptr);
DORIS_CHECK(eof != nullptr);
@@ -368,7 +386,9 @@ Status DelimitedTextReader::get_block(Block* file_block,
size_t* rows, bool* eof
return Status::InternalError("{} v2 reader is not open", _reader_name);
}
- const auto batch_size = _runtime_state != nullptr ?
_runtime_state->batch_size() : 4096;
+ const size_t batch_size =
+ _batch_size != 0 ? _batch_size
+ : (_runtime_state != nullptr ?
_runtime_state->batch_size() : 4096);
const auto max_block_bytes = _runtime_state != nullptr
?
_runtime_state->preferred_block_size_bytes()
: std::numeric_limits<size_t>::max();
@@ -378,6 +398,7 @@ Status DelimitedTextReader::get_block(Block* file_block,
size_t* rows, bool* eof
{
auto columns_guard = file_block->mutate_columns_scoped();
auto& columns = columns_guard.mutable_columns();
+ _prepare_columns(columns, batch_size);
// Delimited text readers are column-pruned but not lazy materialized:
all file-local
// columns requested by TableReader are decoded before file-local
conjuncts are evaluated.
while (*rows < batch_size && !_line_reader_eof &&
@@ -436,6 +457,7 @@ Status DelimitedTextReader::get_block(Block* file_block,
size_t* rows, bool* eof
Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest&
request,
FileAggregateResult* result) {
SCOPED_TIMER(_text_profile.total_time);
+ Defer profile_flush {[this] { _flush_profile(); }};
DORIS_CHECK(result != nullptr);
if (request.agg_type != TPushAggOp::type::COUNT) {
return Status::NotSupported("{} v2 reader only supports COUNT
aggregate pushdown",
@@ -460,7 +482,7 @@ Status DelimitedTextReader::get_aggregate_result(const
FileAggregateRequest& req
break;
}
if (line.size == 0) {
- update_counter(_text_profile.empty_lines_read, 1);
+ ++_pending_profile.empty_lines_read;
if (_empty_line_as_record() ||
(_runtime_state != nullptr &&
_runtime_state->is_read_csv_empty_line_as_null())) {
++count;
@@ -486,6 +508,13 @@ Status DelimitedTextReader::close() {
_line_reader.reset();
}
_decompressor.reset();
+ // Stream pipes are shared with the load context; preserve their existing
ownership and
+ // cancellation behavior. Only filesystem readers enter the sequential
prefetch path.
+ if (_file_reader != nullptr && _scan_params->file_type !=
TFileType::FILE_STREAM) {
+ // Drain prefetch workers before publishing their counters or
releasing IOContext refs.
+ RETURN_IF_ERROR(_file_reader->close());
+ _file_reader->collect_profile_before_close();
+ }
_file_reader.reset();
_tracing_file_reader.reset();
_requested_columns.clear();
@@ -605,8 +634,14 @@ Status DelimitedTextReader::_open_file() {
} else {
auto reader_options =
FileFactory::get_reader_options(_runtime_state->query_options(),
*_file_description);
- auto file_reader = DORIS_TRY(FileFactory::create_file_reader(
- *_system_properties, *_file_description, reader_options,
_profile));
+ // Include the prefix used to align non-first splits. Unknown-size
first splits read to
+ // EOF; PrefetchBufferedReader bounds this range by the opened file's
physical size.
+ const auto prefetch_end = _size < 0 ?
std::numeric_limits<size_t>::max()
+ : cast_set<size_t>(_start_offset +
_size);
+ auto file_reader = DORIS_TRY(io::DelegateReader::create_file_reader(
+ _profile, *_system_properties, *_file_description,
reader_options,
+ io::DelegateReader::AccessMode::SEQUENTIAL, _io_ctx,
+ io::PrefetchRange(_start_offset, prefetch_end)));
_file_reader = _io_ctx && _io_ctx->file_reader_stats
?
std::make_shared<io::TracingFileReader>(std::move(file_reader),
_io_ctx->file_reader_stats)
@@ -624,7 +659,7 @@ Status DelimitedTextReader::_read_next_line(Slice* line,
bool* eof) {
DORIS_CHECK(line != nullptr);
DORIS_CHECK(eof != nullptr);
if (_align_split_prefix) {
- SCOPED_TIMER(_text_profile.read_line_time);
+ SCOPED_TIMER(_text_profile.split_prefix_time);
DCHECK_EQ(_skip_lines, 1);
size_t skipped_lines = 0;
auto* text_reader =
assert_cast<NewPlainTextLineReader*>(_line_reader.get());
@@ -634,13 +669,19 @@ Status DelimitedTextReader::_read_next_line(Slice* line,
bool* eof) {
_align_split_prefix = false;
_skip_lines = 0;
_bom_removed = true;
- update_counter(_text_profile.skipped_lines, skipped_lines);
+ _pending_profile.skipped_lines += skipped_lines;
}
while (true) {
const uint8_t* ptr = nullptr;
size_t size = 0;
{
- SCOPED_TIMER(_text_profile.read_line_time);
+ _sample_current_line =
+ _profile != nullptr && (_read_line_calls++ %
PROFILE_SAMPLE_INTERVAL == 0);
+ std::optional<ScopedRawTimer<MonotonicStopWatch, int64_t>> timer;
+ if (_sample_current_line) {
+ ++_pending_profile.sampled_read_line_calls;
+ timer.emplace(&_pending_profile.read_line_time);
+ }
RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size,
&_line_reader_eof, _io_ctx.get()));
}
if (_line_reader_eof && size == 0) {
@@ -656,12 +697,12 @@ Status DelimitedTextReader::_read_next_line(Slice* line,
bool* eof) {
if (_skip_lines > 0) {
--_skip_lines;
_bom_removed = true;
- update_counter(_text_profile.skipped_lines, 1);
+ ++_pending_profile.skipped_lines;
continue;
}
*line = Slice(ptr, size);
*eof = false;
- update_counter(_text_profile.raw_lines_read, 1);
+ ++_pending_profile.raw_lines_read;
return Status::OK();
}
}
@@ -671,12 +712,12 @@ Status DelimitedTextReader::_fill_columns_from_line(const
Slice& line,
size_t* rows) {
DORIS_CHECK(columns != nullptr);
if (line.size == 0) {
- update_counter(_text_profile.empty_lines_read, 1);
+ ++_pending_profile.empty_lines_read;
if (!_empty_line_as_record()) {
if (_runtime_state != nullptr &&
_runtime_state->is_read_csv_empty_line_as_null()) {
for (const auto& column : _requested_columns) {
RETURN_IF_ERROR(_append_null((*columns)[column.block_position.value()].get()));
- update_counter(_text_profile.cells_deserialized, 1);
+ ++_pending_profile.cells_deserialized;
}
++(*rows);
}
@@ -686,10 +727,17 @@ Status DelimitedTextReader::_fill_columns_from_line(const
Slice& line,
RETURN_IF_ERROR(_validate_line(line));
{
- SCOPED_TIMER(_text_profile.split_line_time);
+ std::optional<ScopedRawTimer<MonotonicStopWatch, int64_t>> timer;
+ if (_sample_current_line) {
+ ++_pending_profile.sampled_parsed_rows;
+ timer.emplace(&_pending_profile.split_line_time);
+ }
_split_line(line);
}
- SCOPED_TIMER(_text_profile.deserialize_time);
+ std::optional<ScopedRawTimer<MonotonicStopWatch, int64_t>> timer;
+ if (_sample_current_line) {
+ timer.emplace(&_pending_profile.deserialize_time);
+ }
if (_request->reject_invalid_rows) {
const auto expected =
cast_set<size_t>(_scan_params->num_of_columns_from_file);
const bool ignore_extra =
_scan_params->file_attributes.ignore_csv_redundant_col;
@@ -708,12 +756,31 @@ Status DelimitedTextReader::_fill_columns_from_line(const
Slice& line,
? _split_values[field_index]
: Slice(_options.null_format, _options.null_len);
RETURN_IF_ERROR(_deserialize_one_cell(column, output,
_normalize_value(value)));
- update_counter(_text_profile.cells_deserialized, 1);
+ ++_pending_profile.cells_deserialized;
}
++(*rows);
return Status::OK();
}
+void DelimitedTextReader::_flush_profile() {
+ update_counter(_text_profile.raw_lines_read,
_pending_profile.raw_lines_read);
+ update_counter(_text_profile.empty_lines_read,
_pending_profile.empty_lines_read);
+ update_counter(_text_profile.skipped_lines,
_pending_profile.skipped_lines);
+ update_counter(_text_profile.cells_deserialized,
_pending_profile.cells_deserialized);
+ update_counter(_text_profile.read_line_time,
_pending_profile.read_line_time);
+ update_counter(_text_profile.split_line_time,
_pending_profile.split_line_time);
+ update_counter(_text_profile.deserialize_time,
_pending_profile.deserialize_time);
+ update_counter(_text_profile.sampled_read_line_calls,
_pending_profile.sampled_read_line_calls);
+ update_counter(_text_profile.sampled_parsed_rows,
_pending_profile.sampled_parsed_rows);
+ _pending_profile = {};
+}
+
+void DelimitedTextReader::_prepare_columns(std::vector<MutableColumnPtr>&
columns,
+ size_t batch_size) {
+ (void)columns;
+ (void)batch_size;
+}
+
Status DelimitedTextReader::_validate_line(const Slice& line) {
(void)line;
return Status::OK();
diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h
b/be/src/format_v2/delimited_text/delimited_text_reader.h
index 645ba26eecd..4b7fc1f4014 100644
--- a/be/src/format_v2/delimited_text/delimited_text_reader.h
+++ b/be/src/format_v2/delimited_text/delimited_text_reader.h
@@ -55,6 +55,7 @@ public:
TableColumnMapperOptions options) const override;
Status open(std::shared_ptr<FileScanRequest> request) override;
Status get_block(Block* file_block, size_t* rows, bool* eof) override;
+ void set_batch_size(size_t batch_size) override;
Status get_aggregate_result(const FileAggregateRequest& request,
FileAggregateResult* result) override;
Status close() override;
@@ -64,6 +65,9 @@ protected:
RuntimeProfile::Counter* total_time = nullptr;
RuntimeProfile::Counter* open_file_time = nullptr;
RuntimeProfile::Counter* create_line_reader_time = nullptr;
+ RuntimeProfile::Counter* split_prefix_time = nullptr;
+ RuntimeProfile::Counter* sampled_read_line_calls = nullptr;
+ RuntimeProfile::Counter* sampled_parsed_rows = nullptr;
RuntimeProfile::Counter* read_line_time = nullptr;
RuntimeProfile::Counter* split_line_time = nullptr;
RuntimeProfile::Counter* deserialize_time = nullptr;
@@ -112,6 +116,8 @@ protected:
// Split one logical line into `_split_values`. The common materialization
path then resolves
// requested field ordinals against `_split_values`.
virtual void _split_line(const Slice& line) = 0;
+ // Prepare mutable output columns once per batch, after Block has resolved
copy-on-write.
+ virtual void _prepare_columns(std::vector<MutableColumnPtr>& columns,
size_t batch_size);
// Deserialize a single normalized field into the requested output column
using the
// format-specific serde API.
virtual Status _deserialize_one_cell(const RequestedColumn& column,
IColumn* output,
@@ -178,6 +184,26 @@ private:
Status _fill_columns_from_line(const Slice& line,
std::vector<MutableColumnPtr>* columns,
size_t* rows);
+ void _flush_profile();
+
+ // Publish exact counts once per API call instead of updating atomic
counters per cell.
+ struct PendingProfile {
+ int64_t raw_lines_read = 0;
+ int64_t empty_lines_read = 0;
+ int64_t skipped_lines = 0;
+ int64_t cells_deserialized = 0;
+ int64_t read_line_time = 0;
+ int64_t split_line_time = 0;
+ int64_t deserialize_time = 0;
+ int64_t sampled_read_line_calls = 0;
+ int64_t sampled_parsed_rows = 0;
+ } _pending_profile;
+ // Keep sampling periodic across batches, including skipped headers and
EOF probes.
+ static constexpr size_t PROFILE_SAMPLE_INTERVAL = 1024;
+ uint64_t _read_line_calls = 0;
+ bool _sample_current_line = false;
+ // Zero means use the runtime default until TableReader supplies its
predicted row cap.
+ size_t _batch_size = 0;
std::string _reader_name;
};
diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
index 91eafdba431..d63f63bc286 100644
--- a/be/src/io/fs/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -413,6 +413,13 @@ void PrefetchBuffer::reset_offset(size_t offset) {
_prefetched.notify_all();
return;
}
+ _merge_file_cache_stats();
+ // Snapshot on the consumer thread, not concurrently with scanner
counter updates.
+ // Cached readers get per-buffer counters; uncached readers keep a
null stats pointer.
+ _prefetch_io_ctx = *_io_ctx;
+ if (_io_ctx->file_cache_stats != nullptr) {
+ _prefetch_io_ctx.file_cache_stats = &_file_cache_stats;
+ }
_buffer_status = BufferStatus::RESET;
_offset = offset;
_prefetched.notify_all();
@@ -470,7 +477,7 @@ void PrefetchBuffer::prefetch_buffer() {
{
SCOPED_RAW_TIMER(&_statis.read_time);
- s = _reader->read_at(_offset, Slice {_buf.data(), buf_size}, &_len,
_io_ctx);
+ s = _reader->read_at(_offset, Slice {_buf.data(), buf_size}, &_len,
&_prefetch_io_ctx);
}
if (UNLIKELY(s.ok() && buf_size != _len)) {
// This indicates that the data size returned by S3 object storage is
smaller than what we requested,
@@ -497,6 +504,7 @@ void PrefetchBuffer::prefetch_buffer() {
_offset, s.to_string());
_prefetch_status = std::move(s);
}
+ _has_unmerged_cache_stats = _prefetch_io_ctx.file_cache_stats != nullptr;
_buffer_status = BufferStatus::PREFETCHED;
_prefetched.notify_all();
// eof would come up with len == 0, it would be handled by read_buffer
@@ -582,6 +590,7 @@ Status PrefetchBuffer::read_buffer(size_t off, const char*
out, size_t buf_len,
if (UNLIKELY(BufferStatus::CLOSED == _buffer_status)) {
return Status::OK();
}
+ _merge_file_cache_stats();
}
RETURN_IF_ERROR(_prefetch_status);
// there is only parquet would do not sequence read
@@ -612,15 +621,23 @@ Status PrefetchBuffer::read_buffer(size_t off, const
char* out, size_t buf_len,
return Status::OK();
}
+void PrefetchBuffer::_merge_file_cache_stats() {
+ // Called only by the consumer with _lock held and no pending read. A
buffer may be
+ // consumed in several slices; publish each completed request exactly once.
+ if (_has_unmerged_cache_stats) {
+ _io_ctx->file_cache_stats->merge_from(_file_cache_stats);
+ _file_cache_stats = {};
+ _has_unmerged_cache_stats = false;
+ }
+}
+
void PrefetchBuffer::close() {
std::unique_lock lck {_lock};
- // in case _reader still tries to write to the buf after we close the
buffer
- if (!_prefetched.wait_for(lck,
-
std::chrono::milliseconds(config::buffered_reader_read_timeout_ms),
- [this]() { return _buffer_status !=
BufferStatus::PENDING; })) {
- _prefetch_status = Status::TimedOut("time out when close prefetch
buffer");
- return;
- }
+ // A foreground read timeout does not cancel the underlying I/O. Wait for
it to finish
+ // before releasing scanner-owned IOContext pointees, including cache
statistics and
+ // the cache-write limiter. Queued tasks observe CLOSED and do not start
another read.
+ _prefetched.wait(lck, [this]() { return _buffer_status !=
BufferStatus::PENDING; });
+ _merge_file_cache_stats();
_buffer_status = BufferStatus::CLOSED;
_prefetched.notify_all();
// Explicitly release the backing buffer here, in the calling (query)
thread which has a
@@ -647,14 +664,14 @@
PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File
}
_io_ctx = _io_ctx_holder.get();
if (buffer_size == -1L) {
- buffer_size = config::remote_storage_read_buffer_mb * 1024 * 1024;
+ buffer_size = int64_t {config::remote_storage_read_buffer_mb} * 1024 *
1024;
}
_size = _reader->size();
- _whole_pre_buffer_size = buffer_size;
+ // Use the allocated capacity as the ring modulus. For example, 6 MiB
allocates
+ // one 4 MiB buffer, so offset 4 MiB must wrap to slot 0, not index a
missing slot 1.
+ const int64_t buffer_num = std::max<int64_t>(1, buffer_size /
s_max_pre_buffer_size);
+ _whole_pre_buffer_size = buffer_num * s_max_pre_buffer_size;
_file_range.end_offset = std::min(_file_range.end_offset, _size);
- int buffer_num = buffer_size > s_max_pre_buffer_size
- ? cast_set<int>(buffer_size) /
cast_set<int>(s_max_pre_buffer_size)
- : 1;
std::function<void(PrefetchBuffer&)> sync_buffer = nullptr;
if (profile != nullptr) {
const char* prefetch_buffered_reader = "PrefetchBufferedReader";
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index 04566bea095..7b7f5031692 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -471,6 +471,11 @@ struct PrefetchBuffer :
std::enable_shared_from_this<PrefetchBuffer>, public Pro
io::FileReaderSPtr _reader;
std::shared_ptr<const IOContext> _io_ctx_holder;
const IOContext* _io_ctx = nullptr;
+ // Only the worker writes these statistics. The consumer merges them after
observing
+ // completion under _lock, before reusing or closing the buffer.
+ IOContext _prefetch_io_ctx;
+ FileCacheStatistics _file_cache_stats;
+ bool _has_unmerged_cache_stats = false;
PODArray<char> _buf;
BufferStatus _buffer_status {BufferStatus::RESET};
std::mutex _lock;
@@ -517,6 +522,9 @@ struct PrefetchBuffer :
std::enable_shared_from_this<PrefetchBuffer>, public Pro
void _collect_profile_at_runtime() override {}
void _collect_profile_before_close() override;
+
+private:
+ void _merge_file_cache_stats();
};
constexpr int64_t s_max_pre_buffer_size = 4 * 1024 * 1024; // 4MB
diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp
b/be/test/format_v2/delimited_text/csv_reader_test.cpp
index 32d4255acee..6c9682ad151 100644
--- a/be/test/format_v2/delimited_text/csv_reader_test.cpp
+++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp
@@ -20,9 +20,12 @@
#include <gtest/gtest.h>
#include <algorithm>
+#include <atomic>
+#include <cstring>
#include <filesystem>
#include <fstream>
#include <memory>
+#include <thread>
#include "common/config.h"
#include "common/consts.h"
@@ -44,17 +47,95 @@
#include "exprs/vexpr_context.h"
#include "format/csv/csv_reader.h"
#include "format_v2/column_mapper.h"
+#include "io/fs/stream_load_pipe.h"
#include "io/io_common.h"
+#include "runtime/exec_env.h"
#include "runtime/runtime_profile.h"
#include "testutil/desc_tbl_builder.h"
#include "testutil/mock/mock_runtime_state.h"
+#include "testutil/mock/obj_storage_client_test_stub.h"
#include "testutil/scoped_temp_dir.h"
#include "util/debug_points.h"
#include "util/defer_op.h"
+#include "util/s3_util.h"
+#include "util/threadpool.h"
namespace doris::format::csv {
namespace {
+class CsvPrefetchS3Client : public io::ObjStorageClientTestStub {
+public:
+ const std::string data =
"id,name,score\n1,alice,10\n2,bob,20\n3,carol,30\n";
+ const std::thread::id scanner_thread = std::this_thread::get_id();
+ std::atomic<size_t> foreground_reads {0};
+ std::atomic<size_t> reads {0};
+
+ io::ObjStorageHeadResult head_object(const io::ObjStoragePath&) override {
+ return {.resp = io::ObjStorageResponse::OK(),
+ .file_size = static_cast<long long>(data.size())};
+ }
+
+ io::ObjStorageResponse get_object(const io::ObjStoragePath&, void* buffer,
size_t offset,
+ size_t bytes_read, size_t* size_return)
override {
+ ++reads;
+ if (std::this_thread::get_id() == scanner_thread) {
+ ++foreground_reads;
+ }
+ EXPECT_LE(offset + bytes_read, data.size());
+ *size_return = std::min(bytes_read, data.size() - offset);
+ std::memcpy(buffer, data.data() + offset, *size_return);
+ return io::ObjStorageResponse::OK();
+ }
+
+ // Only HEAD and GET belong to this read test. Explicit overrides avoid
generating unused
+ // gMock expectation helpers for this anonymous-namespace client.
+ io::ObjStorageUploadResult create_multipart_upload(const
io::ObjStoragePath&) override {
+ return {.resp = unexpected_operation()};
+ }
+
+ io::ObjStorageResponse put_object(const io::ObjStoragePath&,
std::string_view) override {
+ return unexpected_operation();
+ }
+
+ io::ObjStorageUploadResult upload_part(const io::ObjStoragePath&, const
std::string&,
+ std::string_view, int) override {
+ return {.resp = unexpected_operation()};
+ }
+
+ io::ObjStorageResponse complete_multipart_upload(
+ const io::ObjStoragePath&, const std::string&,
+ const std::vector<io::ObjStorageCompletedPart>&) override {
+ return unexpected_operation();
+ }
+
+ io::ObjStorageResponse delete_objects(const io::ObjStoragePath&,
+ std::vector<std::string>) override {
+ return unexpected_operation();
+ }
+
+ io::ObjStorageResponse delete_object(const io::ObjStoragePath&) override {
+ return unexpected_operation();
+ }
+
+ std::string generate_presigned_url(const io::ObjStoragePath&, int64_t)
override {
+ ADD_FAILURE() << "Unexpected signed URL request in CSV prefetch test";
+ return {};
+ }
+
+ io::ObjStorageListPageResult list_objects_page(const io::ObjStoragePath&,
+ std::string_view) override {
+ return {.resp = unexpected_operation()};
+ }
+
+private:
+ static io::ObjStorageResponse unexpected_operation() {
+ ADD_FAILURE() << "Unexpected object storage operation in CSV prefetch
test";
+ return {.status = {TStatusCode::NOT_IMPLEMENTED_ERROR,
+ "CSV prefetch test only supports HEAD and GET"},
+ .http_code = 0};
+ }
+};
+
TFileScanRangeParams csv_scan_params() {
TFileScanRangeParams params;
params.__set_format_type(TFileFormatType::FORMAT_CSV_PLAIN);
@@ -512,6 +593,78 @@ protected:
TFileScanRangeParams _params;
};
+TEST_F(CsvV2ReaderTest, S3PrefetchPreservesSplitRowsAndReadStatistics) {
+ auto* env = ExecEnv::GetInstance();
+ auto saved_pool = std::move(env->_buffered_reader_prefetch_thread_pool);
+ const auto saved_in_memory_size = config::in_memory_file_size;
+ Defer restore([&] {
+ env->_buffered_reader_prefetch_thread_pool.reset();
+ env->_buffered_reader_prefetch_thread_pool = std::move(saved_pool);
+ config::in_memory_file_size = saved_in_memory_size;
+ S3ClientFactory::instance().clear_client_creator_for_test();
+ });
+ ASSERT_TRUE(ThreadPoolBuilder("CsvPrefetchTest")
+ .set_min_threads(1)
+ .set_max_threads(1)
+ .build(&env->_buffered_reader_prefetch_thread_pool)
+ .ok());
+ // Keep the small fixture on the same prefetch path as a large S3 CSV file.
+ config::in_memory_file_size = 0;
+ auto client = std::make_shared<CsvPrefetchS3Client>();
+ S3ClientFactory::instance().set_client_creator_for_test(
+ [client](const S3ClientConf&) { return client; });
+ _params.__set_file_type(TFileType::FILE_S3);
+ _state._query_options.__set_enable_file_cache(false);
+
+ // Cover a complete file, a split starting inside Alice's row, and an
unknown-size first split.
+ for (const int64_t split_start : {0, 17, -1}) {
+ SCOPED_TRACE(split_start);
+ auto properties = std::make_shared<io::FileSystemProperties>();
+ properties->system_type = TFileType::FILE_S3;
+ properties->properties = {{"AWS_ENDPOINT",
"http://csv-prefetch.invalid"},
+ {"AWS_REGION", "test-region"}};
+ auto desc = std::make_unique<io::FileDescription>();
+ desc->path = "s3://csv-prefetch/reader.csv";
+ desc->range_start_offset = std::max<int64_t>(0, split_start);
+ desc->file_size = split_start < 0 ? -1 :
static_cast<int64_t>(client->data.size());
+ desc->range_size = desc->file_size - desc->range_start_offset;
+ io::FileReaderStats stats;
+ auto io_ctx = std::make_shared<io::IOContext>();
+ io_ctx->file_reader_stats = &stats;
+ auto reader =
+ std::make_unique<CsvReader>(properties, desc, io_ctx,
&_profile, &_params, _slots);
+ ASSERT_TRUE(reader->init(&_state).ok());
+ std::vector<ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<FileScanRequest>();
+ for (int32_t id = 0; id < 3; ++id) {
+ request->non_predicate_columns.push_back(
+ LocalColumnIndex::top_level(LocalColumnId(id)));
+ request->local_positions.emplace(LocalColumnId(id),
LocalIndex(id));
+ }
+ ASSERT_TRUE(reader->open(request).ok());
+ auto block = make_block(schema, {0, 1, 2});
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ ASSERT_EQ(rows, split_start > 0 ? 2 : 3);
+ EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0),
split_start > 0 ? 2 : 1);
+ EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, rows -
1), "carol");
+ EXPECT_GT(stats.read_calls, 0);
+ EXPECT_GT(stats.read_bytes, 0);
+ const auto previous_prefetch_io = counter_value(&_profile,
"PreRequestIO");
+ ASSERT_TRUE(reader->close().ok());
+ EXPECT_GT(counter_value(&_profile, "PreRequestIO"),
previous_prefetch_io);
+ EXPECT_GT(counter_value(&_profile, "PreRequestBytes"), 0);
+ EXPECT_GT(counter_value(&_profile, "ReadTime"), 0);
+ const auto collected_prefetch_io = counter_value(&_profile,
"PreRequestIO");
+ ASSERT_TRUE(reader->close().ok());
+ EXPECT_EQ(counter_value(&_profile, "PreRequestIO"),
collected_prefetch_io);
+ }
+ EXPECT_GT(client->reads.load(), 0);
+ EXPECT_EQ(client->foreground_reads.load(), 0);
+}
+
// Scenario: CSV v2 exposes FE-provided file slots as nullable file-local
schema using column_idxs
// as CSV field ordinals.
TEST_F(CsvV2ReaderTest, SchemaUsesSlotTypesAndColumnIdxs) {
@@ -599,9 +752,9 @@ TEST_F(CsvV2ReaderTest,
ProfileCountersTrackReadParseDeserializeAndFilter) {
EXPECT_NE(_profile.get_counter("OpenFileTime"), nullptr);
EXPECT_NE(_profile.get_counter("CreateLineReaderTime"), nullptr);
- EXPECT_NE(_profile.get_counter("ReadLineTime"), nullptr);
- EXPECT_NE(_profile.get_counter("SplitLineTime"), nullptr);
- EXPECT_NE(_profile.get_counter("DeserializeTime"), nullptr);
+ EXPECT_NE(_profile.get_counter("SampledReadLineTime"), nullptr);
+ EXPECT_NE(_profile.get_counter("SampledSplitLineTime"), nullptr);
+ EXPECT_NE(_profile.get_counter("SampledDeserializeTime"), nullptr);
EXPECT_NE(_profile.get_counter("ConjunctFilterTime"), nullptr);
EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr);
EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3);
@@ -616,6 +769,162 @@ TEST_F(CsvV2ReaderTest,
ProfileCountersTrackReadParseDeserializeAndFilter) {
EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6);
}
+// A row cap can change between calls. It must override the runtime default
without dropping
+// records, and a zero prediction must still make progress.
+TEST_F(CsvV2ReaderTest, HonorsChangingBatchSizeAndZeroPrediction) {
+ std::ofstream output(_file_path, std::ios::binary);
+ output << "id,name,score\n";
+ for (int i = 1; i <= 7; ++i) {
+ output << i << ",name,10\n";
+ }
+ output.close();
+ _state._batch_size = 2;
+ auto reader = create_reader(_file_path, &_params, _slots, &_state,
&_profile);
+ reader->set_batch_size(1);
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
+ request->local_positions.emplace(LocalColumnId(1), LocalIndex(1));
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ int next_id = 1;
+ const std::vector<size_t> caps {1, 3, 0, 2, 2};
+ const std::vector<size_t> expected {1, 3, 1, 2, 0};
+ for (size_t i = 0; i < caps.size(); ++i) {
+ reader->set_batch_size(caps[i]);
+ auto block = make_block(schema, {0, 1});
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ ASSERT_EQ(rows, expected[i]);
+ EXPECT_EQ(eof, rows == 0);
+ for (size_t row = 0; row < rows; ++row) {
+ EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, row),
next_id++);
+ EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column,
row), "name");
+ }
+ EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), (next_id - 1)
* 2);
+ EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), next_id - 1);
+ }
+ EXPECT_EQ(next_id, 8);
+ EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1);
+}
+
+TEST_F(CsvV2ReaderTest, BatchPredictionStillHonorsByteBudget) {
+ const bool saved = config::enable_adaptive_batch_size;
+ Defer restore {[&] { config::enable_adaptive_batch_size = saved; }};
+ config::enable_adaptive_batch_size = true;
+ _state._query_options.__set_preferred_block_size_bytes(1);
+ auto reader = create_reader(_file_path, &_params, _slots, &_state,
&_profile);
+ reader->set_batch_size(100);
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(1), LocalIndex(0));
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ for (const std::string expected : {"alice", "bob"}) {
+ auto block = make_block(schema, {1});
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ ASSERT_EQ(rows, 1);
+ EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0),
expected);
+ }
+}
+
+// Sampling spans batch boundaries; exact work counts are published before
every return.
+TEST_F(CsvV2ReaderTest, ProfileSamplingSpansBatchesWithExactCounts) {
+ std::ofstream output(_file_path, std::ios::binary);
+ for (int i = 0; i < 2050; ++i) {
+ output << "1,name,10\n";
+ }
+ output.close();
+ _params.file_attributes.__set_header_type("");
+ auto reader = create_reader(_file_path, &_params, _slots, &_state,
&_profile);
+ reader->set_batch_size(300);
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(1), LocalIndex(0));
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ size_t total = 0;
+ bool eof = false;
+ while (!eof) {
+ auto block = make_block(schema, {1});
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ total += rows;
+ EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), total);
+ EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), total);
+ }
+ EXPECT_EQ(total, 2050);
+ EXPECT_EQ(counter_value(&_profile, "SampledReadLineCalls"), 3);
+ EXPECT_EQ(counter_value(&_profile, "SampledParsedRows"), 3);
+}
+
+// The single-byte split and direct string insertion must preserve NULL,
quoted literals,
+// escaping, trailing empty fields, missing fields and reordered projections
across batches.
+TEST_F(CsvV2ReaderTest,
StringFastPathPreservesValuesForSingleAndMultiByteSeparators) {
+ auto type = make_nullable(std::make_shared<DataTypeString>());
+ std::vector<SlotDescriptor*> slots;
+ for (int i = 0; i < 3; ++i) {
+ slots.push_back(make_test_slot(&_pool, i, i, type, "s" +
std::to_string(i)));
+ }
+ for (const std::string separator : {"|", "||"}) {
+ for (bool empty_as_null : {false, true}) {
+ SCOPED_TRACE(testing::Message() << separator << ", empty_as_null="
<< empty_as_null);
+ std::ofstream output(_file_path, std::ios::binary);
+ output << "NULL" << separator << "\"NULL\"" << separator << "\n";
+ output << "left" << separator << "a\\b" << separator << "tail\n";
+ output << "only\n";
+ output.close();
+ auto params = _params;
+ params.file_attributes.__set_header_type("");
+ params.file_attributes.__set_trim_double_quotes(true);
+
params.file_attributes.text_params.__set_column_separator(separator);
+ params.file_attributes.text_params.__set_null_format("NULL");
+
params.file_attributes.text_params.__set_empty_field_as_null(empty_as_null);
+ params.file_attributes.text_params.__set_escape('\\');
+ auto reader = create_reader(_file_path, ¶ms, slots, &_state,
&_profile);
+ reader->set_batch_size(1);
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(2), LocalIndex(0));
+ request->local_positions.emplace(LocalColumnId(0), LocalIndex(1));
+ request->local_positions.emplace(LocalColumnId(1), LocalIndex(2));
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ for (int row = 0; row < 3; ++row) {
+ auto block = make_block(schema, {2, 0, 1});
+ // Hold shared columns while the reader acquires its mutable
batch.
+ auto shared = block.get_by_position(0).column;
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ ASSERT_EQ(rows, 1);
+ EXPECT_EQ(shared->size(), 0);
+ const auto& c0 = *block.get_by_position(0).column;
+ const auto& c1 = *block.get_by_position(1).column;
+ const auto& c2 = *block.get_by_position(2).column;
+ if (row == 0) {
+ EXPECT_EQ(is_null_at(c0, 0), empty_as_null);
+ EXPECT_EQ(nullable_string_at(c0, 0), "");
+ EXPECT_TRUE(is_null_at(c1, 0));
+ EXPECT_FALSE(is_null_at(c2, 0));
+ EXPECT_EQ(nullable_string_at(c2, 0), "NULL");
+ } else if (row == 1) {
+ EXPECT_EQ(nullable_string_at(c0, 0), "tail");
+ EXPECT_EQ(nullable_string_at(c1, 0), "left");
+ EXPECT_EQ(nullable_string_at(c2, 0), "ab");
+ } else {
+ EXPECT_TRUE(is_null_at(c0, 0));
+ EXPECT_EQ(nullable_string_at(c1, 0), "only");
+ EXPECT_TRUE(is_null_at(c2, 0));
+ }
+ }
+ }
+ }
+}
+
// Scenario: CSV has no embedded nested schema, but TableColumnMapper still
needs semantic children
// for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children
from the slot type
// while keeping the top-level local id as the CSV field ordinal from
column_idxs.
@@ -675,6 +984,8 @@ TEST_F(CsvV2ReaderTest,
LoadRejectsWrongFieldCountsWithoutShiftingFollowingRows)
EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 1);
EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 1), 4);
EXPECT_EQ(io_ctx->load_filtered_rows, 2);
+ EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 4);
+ EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6);
}
// Scenario: CSV v2 honors FileScanRequest local positions, so TableReader can
request a subset of
@@ -727,6 +1038,9 @@ TEST_F(CsvV2ReaderTest,
InvalidUtf8FailsWhenValidationEnabled) {
bool eof = false;
const auto status = reader->get_block(&block, &rows, &eof);
EXPECT_FALSE(status.ok());
+ EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 1);
+ EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1);
+ EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 0);
EXPECT_TRUE(status.to_string().find("Only support csv data in utf8 codec")
!= std::string::npos)
<< status;
}
@@ -927,6 +1241,17 @@ TEST_F(CsvV2ReaderTest,
StreamInputRequiresLoadIdBeforeOpeningPipe) {
<< status;
}
+TEST_F(CsvV2ReaderTest, ClosePreservesSharedStreamPipe) {
+ _params.__set_file_type(TFileType::FILE_STREAM);
+ auto reader = create_unknown_size_reader(_file_path, &_params, _slots,
&_state, &_profile);
+ auto pipe = std::make_shared<io::StreamLoadPipe>();
+ reader->_file_reader = pipe;
+ ASSERT_TRUE(reader->close().ok());
+ EXPECT_FALSE(pipe->closed());
+ reader.reset();
+ EXPECT_FALSE(pipe->closed());
+}
+
// Scenario: CSV has no footer row count, so v2 COUNT pushdown scans the split
and returns the
// counted row count through FileAggregateResult.
TEST_F(CsvV2ReaderTest, CountAggregateScansRows) {
@@ -939,6 +1264,10 @@ TEST_F(CsvV2ReaderTest, CountAggregateScansRows) {
FileAggregateResult aggregate_result;
ASSERT_TRUE(reader->get_aggregate_result(aggregate_request,
&aggregate_result).ok());
EXPECT_EQ(aggregate_result.count, 2);
+ EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 2);
+ EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1);
+ EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 0);
+ EXPECT_EQ(counter_value(&_profile, "SampledParsedRows"), 0);
}
TEST_F(CsvV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) {
diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp
b/be/test/format_v2/delimited_text/text_reader_test.cpp
index 4927ee7622e..2fe1aa84bef 100644
--- a/be/test/format_v2/delimited_text/text_reader_test.cpp
+++ b/be/test/format_v2/delimited_text/text_reader_test.cpp
@@ -408,9 +408,9 @@ TEST_F(TextV2ReaderTest,
ProfileCountersTrackReadParseDeserializeAndFilter) {
EXPECT_NE(_profile.get_counter("OpenFileTime"), nullptr);
EXPECT_NE(_profile.get_counter("CreateLineReaderTime"), nullptr);
- EXPECT_NE(_profile.get_counter("ReadLineTime"), nullptr);
- EXPECT_NE(_profile.get_counter("SplitLineTime"), nullptr);
- EXPECT_NE(_profile.get_counter("DeserializeTime"), nullptr);
+ EXPECT_NE(_profile.get_counter("SampledReadLineTime"), nullptr);
+ EXPECT_NE(_profile.get_counter("SampledSplitLineTime"), nullptr);
+ EXPECT_NE(_profile.get_counter("SampledDeserializeTime"), nullptr);
EXPECT_NE(_profile.get_counter("ConjunctFilterTime"), nullptr);
EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr);
EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3);
@@ -424,6 +424,32 @@ TEST_F(TextV2ReaderTest,
ProfileCountersTrackReadParseDeserializeAndFilter) {
EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6);
}
+TEST_F(TextV2ReaderTest, HonorsBatchSizeAcrossEmptyAndFilteredBlocks) {
+ auto reader = create_reader(_file_path, &_params, _slots, &_state,
&_profile);
+ reader->set_batch_size(1);
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
+ request->conjuncts = {
+ prepared_conjunct(&_state,
std::make_shared<NullableIntGreaterThanExpr>(0, 1))};
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ for (int batch = 0; batch < 3; ++batch) {
+ auto block = make_block(schema, {0});
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ EXPECT_EQ(rows, batch == 1 ? 1 : 0);
+ EXPECT_EQ(eof, batch == 2);
+ if (rows == 1) {
+ EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2);
+ }
+ }
+ EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 2);
+ EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 2);
+ EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 1);
+}
+
TEST_F(TextV2ReaderTest,
FormatProfilesKeepDistinctCountersInBothInitializationOrders) {
auto expect_format_children = [](RuntimeProfile* profile) {
TRuntimeProfileTree tree;
diff --git a/be/test/io/fs/buffered_reader_test.cpp
b/be/test/io/fs/buffered_reader_test.cpp
index 75ed5e351dc..b8f96b339fc 100644
--- a/be/test/io/fs/buffered_reader_test.cpp
+++ b/be/test/io/fs/buffered_reader_test.cpp
@@ -20,9 +20,12 @@
#include <glog/logging.h>
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
#include <limits.h>
+#include <algorithm>
#include <atomic>
+#include <cstring>
#include <memory>
#include <ostream>
#include <thread>
@@ -33,6 +36,7 @@
#include "io/fs/local_file_system.h"
#include "runtime/exec_env.h"
#include "util/countdown_latch.h"
+#include "util/defer_op.h"
#include "util/stopwatch.hpp"
#include "util/threadpool.h"
@@ -150,7 +154,7 @@ public:
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
- const io::IOContext* /*io_ctx*/) override {
+ const io::IOContext* io_ctx) override {
_read_started->count_down();
_continue_read->wait();
if (offset >= _size) {
@@ -158,7 +162,9 @@ protected:
return Status::OK();
}
*bytes_read = std::min(_size - offset, result.size);
- return Status::TimedOut("injected prefetch timeout");
+ memset(result.data, 0, *bytes_read);
+ io_ctx->file_cache_stats->bytes_read_from_remote += *bytes_read;
+ return Status::OK();
}
private:
@@ -352,37 +358,160 @@ TEST_F(BufferedReaderTest, test_miss) {
EXPECT_EQ(45, bytes_read);
}
-TEST_F(BufferedReaderTest,
prefetch_task_keeps_reader_alive_after_close_timeout) {
+// A read timeout must not let close/destruction release the scanner's
borrowed state.
+TEST_F(BufferedReaderTest, prefetch_close_drains_timed_out_reads) {
const auto saved_timeout_ms = config::buffered_reader_read_timeout_ms;
- config::buffered_reader_read_timeout_ms = 50;
-
- CountDownLatch read_started(1);
- CountDownLatch continue_read(1);
- std::atomic<bool> reader_destroyed = false;
- auto inner_reader = std::make_shared<BlockingFileReader>(1024,
&read_started, &continue_read,
-
&reader_destroyed);
- auto weak_reader = std::weak_ptr<io::FileReader>(inner_reader);
- {
- io::PrefetchBufferedReader reader(nullptr, std::move(inner_reader),
- io::PrefetchRange(0, 1024));
+ Defer restore([&] { config::buffered_reader_read_timeout_ms =
saved_timeout_ms; });
+ config::buffered_reader_read_timeout_ms = 20;
+
+ for (bool explicit_close : {false, true}) {
+ SCOPED_TRACE(explicit_close);
+ CountDownLatch read_started(1);
+ CountDownLatch continue_read(1);
+ CountDownLatch close_started(1);
+ CountDownLatch close_finished(1);
+ std::atomic<bool> reader_destroyed = false;
+ io::FileCacheStatistics stats;
+ auto io_ctx = std::make_shared<io::IOContext>();
+ io_ctx->file_cache_stats = &stats;
+ auto inner_reader = std::make_shared<BlockingFileReader>(1024,
&read_started,
+
&continue_read, &reader_destroyed);
+ auto reader = std::make_unique<io::PrefetchBufferedReader>(
+ nullptr, std::move(inner_reader), io::PrefetchRange(0, 1024),
io_ctx);
uint8_t buf[1];
size_t bytes_read = 0;
- Status st = reader.read_at(0, Slice {buf, 1}, &bytes_read);
- EXPECT_TRUE(st.is<ErrorCode::TIMEOUT>());
- EXPECT_TRUE(read_started.wait_for(std::chrono::seconds(1)));
- EXPECT_FALSE(reader_destroyed.load());
+ EXPECT_TRUE(reader->read_at(0, Slice {buf, 1},
&bytes_read).is<ErrorCode::TIMEOUT>());
+ EXPECT_TRUE(read_started.wait_for(std::chrono::seconds(5)));
+ EXPECT_EQ(stats.bytes_read_from_remote, 0);
+
+ std::thread finish_read([&] {
+ close_started.wait();
+ // Hold the request beyond the old close timeout. Closing must
remain blocked.
+
EXPECT_FALSE(close_finished.wait_for(std::chrono::milliseconds(100)));
+ EXPECT_FALSE(reader_destroyed.load());
+ continue_read.count_down();
+ });
+ close_started.count_down();
+ if (explicit_close) {
+ EXPECT_TRUE(reader->close().ok());
+ }
+ reader.reset();
+ close_finished.count_down();
+ finish_read.join();
+ // Let the pool release its task captures before the
destructor-observer goes out of scope.
+ ExecEnv::GetInstance()->buffered_reader_prefetch_thread_pool()->wait();
+ EXPECT_TRUE(reader_destroyed.load());
+ EXPECT_EQ(stats.bytes_read_from_remote, 1024);
}
- EXPECT_FALSE(reader_destroyed.load());
- EXPECT_FALSE(weak_reader.expired());
+}
- continue_read.count_down();
- for (int i = 0; i < 100 && !reader_destroyed.load(); ++i) {
- std::this_thread::sleep_for(std::chrono::milliseconds(10));
+class PrefetchStatisticsFileReader : public MockOffsetFileReader {
+public:
+ PrefetchStatisticsFileReader(size_t size, io::FileCacheStatistics*
scanner_stats,
+ CountDownLatch* started, CountDownLatch*
resume)
+ : MockOffsetFileReader(size),
+ _scanner_stats(scanner_stats),
+ _started(started),
+ _resume(resume) {}
+
+ std::atomic<bool> used_scanner_stats = false;
+
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const io::IOContext* io_ctx) override {
+ if (io_ctx->file_cache_stats == _scanner_stats) {
+ used_scanner_stats = true;
+ }
+ RETURN_IF_ERROR(MockOffsetFileReader::read_at_impl(offset, result,
bytes_read, io_ctx));
+ auto* stats = io_ctx->file_cache_stats;
+ stats->bytes_read_from_remote += *bytes_read;
+ ++stats->num_remote_io_total;
+ stats->peer_hosts.insert("prefetch-test");
+ stats->remote_only_on_miss_triggered = true;
+ stats->remote_only_on_miss_threshold_bytes = 42;
+ _started->count_down();
+ _resume->wait();
+ return Status::OK();
}
- EXPECT_TRUE(reader_destroyed.load());
- EXPECT_TRUE(weak_reader.expired());
- config::buffered_reader_read_timeout_ms = saved_timeout_ms;
+private:
+ io::FileCacheStatistics* _scanner_stats;
+ CountDownLatch* _started;
+ CountDownLatch* _resume;
+};
+
+TEST_F(BufferedReaderTest,
prefetch_merges_worker_cache_statistics_on_consumer) {
+ constexpr size_t buffer_size = 4 * 1024 * 1024;
+ constexpr size_t file_size = 5 * buffer_size;
+ CountDownLatch started(4);
+ CountDownLatch resume(1);
+ io::FileCacheStatistics stats;
+ auto io_ctx = std::make_shared<io::IOContext>();
+ io_ctx->file_cache_stats = &stats;
+ auto inner =
+ std::make_shared<PrefetchStatisticsFileReader>(file_size, &stats,
&started, &resume);
+ io::PrefetchBufferedReader reader(nullptr, inner, io::PrefetchRange(0,
file_size), io_ctx,
+ 4 * buffer_size);
+ std::thread release_workers([&] {
+ EXPECT_TRUE(started.wait_for(std::chrono::seconds(5)));
+ // All four workers have updated their counters, but the consumer has
not merged them.
+ EXPECT_FALSE(inner->used_scanner_stats.load());
+ EXPECT_EQ(stats.bytes_read_from_remote, 0);
+ resume.count_down();
+ });
+ uint8_t buf[2];
+ size_t bytes_read = 0;
+ auto status = reader.read_at(0, Slice {buf, 1}, &bytes_read);
+ release_workers.join();
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(stats.bytes_read_from_remote, buffer_size);
+ ASSERT_TRUE(reader.read_at(1, Slice {buf, 1}, &bytes_read).ok());
+ EXPECT_EQ(stats.bytes_read_from_remote, buffer_size); // No double merge
for another slice.
+ // Reuse slot 0 without consuming slots 1..3; close must still collect
their statistics.
+ ASSERT_TRUE(reader.read_at(4 * buffer_size, Slice {buf, 1},
&bytes_read).ok());
+ EXPECT_EQ(stats.bytes_read_from_remote, 2 * buffer_size);
+ ASSERT_TRUE(reader.close().ok());
+ EXPECT_EQ(stats.bytes_read_from_remote, file_size);
+ EXPECT_EQ(stats.num_remote_io_total, 5);
+ EXPECT_EQ(stats.peer_hosts.count("prefetch-test"), 1);
+ EXPECT_TRUE(stats.remote_only_on_miss_triggered);
+ EXPECT_EQ(stats.remote_only_on_miss_threshold_bytes, 42);
+ ASSERT_TRUE(reader.close().ok());
+ EXPECT_EQ(stats.bytes_read_from_remote, file_size);
+}
+
+TEST_F(BufferedReaderTest, prefetch_normalizes_ring_capacity) {
+ constexpr size_t mb = 1024 * 1024;
+ constexpr size_t file_size = 10 * mb;
+ std::vector<uint8_t> expected(file_size);
+ for (size_t i = 0; i < file_size; ++i) {
+ expected[i] = i % UCHAR_MAX;
+ }
+ const auto saved_buffer_mb = config::remote_storage_read_buffer_mb;
+ Defer restore([&] { config::remote_storage_read_buffer_mb =
saved_buffer_mb; });
+ // Includes less than one buffer, a non-multiple of 4 MiB, and a
multi-slot ring.
+ for (int capacity_mb : {0, 1, 6, 10, 16}) {
+ config::remote_storage_read_buffer_mb = capacity_mb;
+ for (size_t start : {size_t {0}, 4 * mb + 7}) {
+ SCOPED_TRACE(capacity_mb);
+ SCOPED_TRACE(start);
+ io::PrefetchBufferedReader reader(nullptr,
+
std::make_shared<MockOffsetFileReader>(file_size),
+ io::PrefetchRange(start,
file_size));
+ std::vector<uint8_t> data(3 * mb);
+ for (size_t offset = start; offset < file_size;) {
+ size_t bytes_read = 0;
+ ASSERT_TRUE(
+ reader.read_at(offset, Slice {data.data(),
data.size()}, &bytes_read).ok());
+ ASSERT_EQ(bytes_read, std::min(data.size(), file_size -
offset));
+ ASSERT_TRUE(std::equal(data.begin(), data.begin() + bytes_read,
+ expected.begin() + offset))
+ << offset;
+ offset += bytes_read;
+ }
+ ASSERT_TRUE(reader.close().ok());
+ }
+ }
}
TEST_F(BufferedReaderTest, test_read_amplify) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]