This is an automated email from the ASF dual-hosted git repository. mrhhsg pushed a commit to branch adaptive_batch_size in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7d63a11f9e4873bfd109d850bfda4b15d64275c6 Author: Hu Shenggang <[email protected]> AuthorDate: Thu Apr 9 15:26:42 2026 +0800 remove config::load_reader_max_block_bytes --- be/src/common/config.cpp | 4 ---- be/src/common/config.h | 4 ---- be/src/format/orc/vorc_reader.cpp | 20 -------------------- be/src/format/orc/vorc_reader.h | 5 +---- be/src/format/parquet/vparquet_reader.cpp | 17 ----------------- be/src/format/parquet/vparquet_reader.h | 5 +---- 6 files changed, 2 insertions(+), 53 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index cfe3939bb05..18829b7089c 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1729,10 +1729,6 @@ DEFINE_mBool(enable_auto_clone_on_mow_publish_missing_version, "false"); // The maximum csv line reader output buffer size DEFINE_mInt64(max_csv_line_reader_output_buffer_size, "4294967296"); -// The maximum bytes of a single block returned by load file readers (CsvReader, NewJsonReader, -// ParquetReader, OrcReader). Default is 64MB. Set to 0 to disable the limit. -DEFINE_mInt64(load_reader_max_block_bytes, "67108864"); - // Maximum number of OpenMP threads allowed for concurrent vector index builds. // -1 means auto: use 80% of the available CPU cores. DEFINE_Int32(omp_threads_limit, "-1"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 7332edc4f32..0416b51b9ff 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1790,10 +1790,6 @@ DECLARE_String(fuzzy_test_type); // The maximum csv line reader output buffer size DECLARE_mInt64(max_csv_line_reader_output_buffer_size); -// The maximum bytes of a single block returned by load file readers (CsvReader, NewJsonReader, -// ParquetReader, OrcReader). Default is 200MB. Set to 0 to disable the limit. -DECLARE_mInt64(load_reader_max_block_bytes); - // Maximum number of OpenMP threads available for concurrent index builds. // -1 means auto: use 80% of detected CPU cores. DECLARE_Int32(omp_threads_limit); diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index ca8664b2577..e39befd0fba 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -2451,14 +2451,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo return Status::OK(); } - // Limit memory per batch for load paths: pre-shrink _batch_size using the bytes-per-row - // estimate from the previous batch so the current batch stays within load_reader_max_block_bytes - // (effective from call 2 onward; first batch is capped on the next call). - const int64_t max_block_bytes = - (_state != nullptr && _state->query_type() == TQueryType::LOAD && - config::load_reader_max_block_bytes > 0) - ? config::load_reader_max_block_bytes - : 0; if (_push_down_agg_type == TPushAggOp::type::COUNT) { auto rows = std::min(get_remaining_rows(), (int64_t)_batch_size); @@ -2475,15 +2467,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo return Status::OK(); } - if (max_block_bytes > 0 && _load_bytes_per_row > 0 && _row_reader) { - size_t new_batch_size = std::max( - (size_t)1, (size_t)((int64_t)max_block_bytes / (int64_t)_load_bytes_per_row)); - if (new_batch_size != _batch_size) { - _batch_size = new_batch_size; - _batch = _row_reader->createRowBatch(_batch_size); - } - } - if (!_seek_to_read_one_line()) { *eof = true; return Status::OK(); @@ -2826,9 +2809,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo *read_rows = block->rows(); } - if (max_block_bytes > 0 && *read_rows > 0) { - _load_bytes_per_row = block->bytes() / *read_rows; - } return Status::OK(); } diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 5c0fd16dd33..47e4425c569 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -694,10 +694,7 @@ private: io::FileSystemProperties _system_properties; io::FileDescription _file_description; size_t _batch_size; - // Bytes-per-row estimate from the previous batch, used to pre-shrink _batch_size - // before reading so that oversized blocks are prevented from the current call onward. - // Zero means no prior data (first batch). - size_t _load_bytes_per_row = 0; + int64_t _range_start_offset; int64_t _range_size; std::string _ctz; diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 4f1648881f8..5685e7b363e 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -727,19 +727,6 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof) return Status::OK(); } - // Limit memory per batch for load paths. - // _load_bytes_per_row is updated after each batch so the *next* call pre-shrinks _batch_size - // before reading, ensuring the current batch is already within the limit (from call 2 onward). - const int64_t max_block_bytes = - (_state != nullptr && _state->query_type() == TQueryType::LOAD && - config::load_reader_max_block_bytes > 0) - ? config::load_reader_max_block_bytes - : 0; - if (max_block_bytes > 0 && _load_bytes_per_row > 0) { - _batch_size = std::max((size_t)1, - (size_t)((int64_t)max_block_bytes / (int64_t)_load_bytes_per_row)); - } - SCOPED_RAW_TIMER(&_reader_statistics.column_read_time); Status batch_st = _current_group_reader->next_batch(block, _batch_size, read_rows, &_row_group_eof); @@ -756,10 +743,6 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof) batch_st.to_string()); } - if (max_block_bytes > 0 && *read_rows > 0) { - _load_bytes_per_row = block->bytes() / *read_rows; - } - if (_row_group_eof) { auto column_st = _current_group_reader->merged_column_statistics(); _column_statistics.merge(column_st); diff --git a/be/src/format/parquet/vparquet_reader.h b/be/src/format/parquet/vparquet_reader.h index c4a7a6a341c..9cad942b856 100644 --- a/be/src/format/parquet/vparquet_reader.h +++ b/be/src/format/parquet/vparquet_reader.h @@ -343,10 +343,7 @@ private: // parquet file reader object size_t _batch_size; - // Bytes-per-row estimate from the previous batch, used to pre-shrink _batch_size - // before reading so that oversized blocks are prevented from the current call onward. - // Zero means no prior data (first batch). - size_t _load_bytes_per_row = 0; + int64_t _range_start_offset; int64_t _range_size; const cctz::time_zone* _ctz = nullptr; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
