This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 51b4567523e branch-3.1: [fix](iceberg) support reading iceberg
partition table without partition value stored in file #56918 (#57043)
51b4567523e is described below
commit 51b4567523ec8f70be080e50226bae017ee16d3c
Author: Socrates <[email protected]>
AuthorDate: Thu Oct 16 23:30:45 2025 +0800
branch-3.1: [fix](iceberg) support reading iceberg partition table without
partition value stored in file #56918 (#57043)
bp: #56918
---
be/src/common/config.cpp | 3 +++
be/src/common/config.h | 3 +++
be/src/vec/exec/scan/vfile_scanner.cpp | 11 +++++++++++
3 files changed, 17 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 91d8c021b16..74bd707704d 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1484,6 +1484,9 @@ DEFINE_mBool(skip_loading_stale_rowset_meta, "false");
DEFINE_Bool(enable_file_logger, "true");
+// Enable partition column fallback when partition columns are missing from
file
+DEFINE_Bool(enable_iceberg_partition_column_fallback, "true");
+
// The minimum row group size when exporting Parquet files. default 128MB
DEFINE_Int64(min_row_group_size, "134217728");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 3c3ec9ac02d..5aa4f8aef86 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1552,6 +1552,9 @@ DECLARE_mBool(skip_loading_stale_rowset_meta);
// Only works when starting BE with --console.
DECLARE_Bool(enable_file_logger);
+// Enable partition column fallback when partition columns are missing from
file
+DECLARE_Bool(enable_iceberg_partition_column_fallback);
+
// The minimum row group size when exporting Parquet files.
DECLARE_Int64(min_row_group_size);
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp
b/be/src/vec/exec/scan/vfile_scanner.cpp
index 54f9c807989..61f32af56e6 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -1317,6 +1317,17 @@ Status VFileScanner::_get_next_reader() {
_slot_lower_name_to_col_type.emplace(to_lower(col_name), col_type);
}
+ if (!_fill_partition_from_path &&
config::enable_iceberg_partition_column_fallback) {
+ // check if the cols of _partition_col_descs are in _missing_cols
+ // if so, set _fill_partition_from_path to true and remove the col
from _missing_cols
+ for (const auto& [col_name, col_type] : _partition_col_descs) {
+ if (_missing_cols.contains(col_name)) {
+ _fill_partition_from_path = true;
+ _missing_cols.erase(col_name);
+ }
+ }
+ }
+
RETURN_IF_ERROR(_generate_missing_columns());
if (_fill_partition_from_path) {
RETURN_IF_ERROR(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
