This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a4d37d96cae [opt](file-scanner) add not found file number in profile
(#37042) (#37764)
a4d37d96cae is described below
commit a4d37d96caec4988a03448d0666a16eb83ec1767
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon Jul 15 17:11:06 2024 +0800
[opt](file-scanner) add not found file number in profile (#37042) (#37764)
bp #37042
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 4 ++++
be/src/vec/exec/scan/vfile_scanner.cpp | 8 ++++++--
be/src/vec/exec/scan/vfile_scanner.h | 1 +
4 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 17bca94a459..523df70a2ad 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1253,6 +1253,8 @@ DEFINE_mInt64(fetch_remote_schema_rpc_timeout_ms,
"60000");
// filter wrong data.
DEFINE_mBool(enable_parquet_page_index, "true");
+DEFINE_mBool(ignore_not_found_file_in_external_table, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 5fa5b23e977..8de25205b62 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1339,6 +1339,10 @@ DECLARE_Int64(min_row_group_size);
DECLARE_mBool(enable_parquet_page_index);
+// Wheather to ignore not found file in external teble(eg, hive)
+// Default is true, if set to false, the not found file will result in query
failure.
+DECLARE_mBool(ignore_not_found_file_in_external_table);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp
b/be/src/vec/exec/scan/vfile_scanner.cpp
index dcfb404ae5a..944884434f4 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -162,6 +162,8 @@ Status VFileScanner::prepare(
_convert_to_output_block_timer =
ADD_TIMER(_parent->_scanner_profile,
"FileScannerConvertOuputBlockTime");
_empty_file_counter = ADD_COUNTER(_parent->_scanner_profile,
"EmptyFileNum", TUnit::UNIT);
+ _not_found_file_counter =
+ ADD_COUNTER(_parent->_scanner_profile, "NotFoundFileNum",
TUnit::UNIT);
_file_counter = ADD_COUNTER(_parent->_scanner_profile, "FileNumber",
TUnit::UNIT);
_has_fully_rf_file_counter =
ADD_COUNTER(_parent->_scanner_profile, "HasFullyRfFileNumber",
TUnit::UNIT);
@@ -180,6 +182,8 @@ Status VFileScanner::prepare(
ADD_TIMER(_local_state->scanner_profile(),
"FileScannerConvertOuputBlockTime");
_empty_file_counter =
ADD_COUNTER(_local_state->scanner_profile(), "EmptyFileNum",
TUnit::UNIT);
+ _not_found_file_counter =
+ ADD_COUNTER(_local_state->scanner_profile(),
"NotFoundFileNum", TUnit::UNIT);
_file_counter = ADD_COUNTER(_local_state->scanner_profile(),
"FileNumber", TUnit::UNIT);
_has_fully_rf_file_counter =
ADD_COUNTER(_local_state->scanner_profile(),
"HasFullyRfFileNumber", TUnit::UNIT);
@@ -329,9 +333,9 @@ Status VFileScanner::_get_block_wrapped(RuntimeState*
state, Block* block, bool*
// And the file may already be removed from storage.
// Just ignore not found files.
Status st = _get_next_reader();
- if (st.is<ErrorCode::NOT_FOUND>()) {
+ if (st.is<ErrorCode::NOT_FOUND>() &&
config::ignore_not_found_file_in_external_table) {
_cur_reader_eof = true;
- COUNTER_UPDATE(_empty_file_counter, 1);
+ COUNTER_UPDATE(_not_found_file_counter, 1);
continue;
} else if (!st) {
return st;
diff --git a/be/src/vec/exec/scan/vfile_scanner.h
b/be/src/vec/exec/scan/vfile_scanner.h
index 43c1a8b13da..2c2116172c9 100644
--- a/be/src/vec/exec/scan/vfile_scanner.h
+++ b/be/src/vec/exec/scan/vfile_scanner.h
@@ -182,6 +182,7 @@ private:
RuntimeProfile::Counter* _pre_filter_timer = nullptr;
RuntimeProfile::Counter* _convert_to_output_block_timer = nullptr;
RuntimeProfile::Counter* _empty_file_counter = nullptr;
+ RuntimeProfile::Counter* _not_found_file_counter = nullptr;
RuntimeProfile::Counter* _file_counter = nullptr;
RuntimeProfile::Counter* _has_fully_rf_file_counter = nullptr;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
