This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new aa9162840f1 [fix](iceberg) Add missing Iceberg field IDs for position
delete files. (#63483)
aa9162840f1 is described below
commit aa9162840f154159943a444ac222d70d69a7a0c2
Author: daidai <[email protected]>
AuthorDate: Tue Jun 2 14:18:45 2026 +0800
[fix](iceberg) Add missing Iceberg field IDs for position delete files.
(#63483)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../writer/iceberg/viceberg_delete_file_writer.cpp | 25 ++++++++++++++++++----
.../writer/iceberg/viceberg_delete_file_writer.h | 4 ++++
2 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
index 8d1724a4396..968ac987e9a 100644
--- a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
+++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
@@ -19,6 +19,8 @@
#include <fmt/format.h>
+#include "format/table/iceberg/schema.h"
+#include "format/table/iceberg/types.h"
#include "format/transformer/vorc_transformer.h"
#include "format/transformer/vparquet_transformer.h"
#include "io/file_factory.h"
@@ -26,6 +28,20 @@
namespace doris {
+// Iceberg reserved field IDs for position delete files.
+constexpr int POSITION_DELETE_FILE_PATH_ID = 2147483546;
+constexpr int POSITION_DELETE_POS_ID = 2147483545;
+
+std::unique_ptr<iceberg::Schema> build_position_delete_schema() {
+ std::vector<iceberg::NestedField> fields;
+ fields.reserve(2);
+ fields.emplace_back(false, POSITION_DELETE_FILE_PATH_ID, "file_path",
+ std::make_unique<iceberg::StringType>(), std::nullopt);
+ fields.emplace_back(false, POSITION_DELETE_POS_ID, "pos",
std::make_unique<iceberg::LongType>(),
+ std::nullopt);
+ return std::make_unique<iceberg::Schema>(std::move(fields));
+}
+
VIcebergDeleteFileWriter::VIcebergDeleteFileWriter(TFileContent::type
delete_type,
const std::string&
output_path,
TFileFormatType::type
file_format,
@@ -46,6 +62,7 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state,
RuntimeProfile* profi
if (_delete_type != TFileContent::POSITION_DELETES) {
return Status::NotSupported("Iceberg delete file writer only supports
position deletes");
}
+ _position_delete_schema = build_position_delete_schema();
_state = state;
@@ -83,15 +100,15 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state,
RuntimeProfile* profi
ParquetFileOptions parquet_options = {parquet_compression_type,
TParquetVersion::PARQUET_1_0,
false, false};
- _file_format_transformer.reset(new VParquetTransformer(state,
_file_writer.get(),
- output_exprs,
column_names, false,
-
parquet_options, nullptr, nullptr));
+ _file_format_transformer.reset(new VParquetTransformer(
+ state, _file_writer.get(), output_exprs, column_names, false,
parquet_options,
+ nullptr, _position_delete_schema.get()));
return _file_format_transformer->open();
}
case TFileFormatType::FORMAT_ORC: {
_file_format_transformer.reset(new VOrcTransformer(state,
_file_writer.get(), output_exprs,
"", column_names,
false, _compress_type,
- nullptr));
+
_position_delete_schema.get()));
return _file_format_transformer->open();
}
default:
diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
index e5de7143f2b..c242731dc15 100644
--- a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
+++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
@@ -32,6 +32,9 @@ namespace doris {
class RuntimeState;
class RuntimeProfile;
class ObjectPool;
+namespace iceberg {
+class Schema;
+}
namespace io {
class FileSystem;
@@ -103,6 +106,7 @@ private:
RuntimeState* _state = nullptr;
std::shared_ptr<io::FileSystem> _fs;
io::FileWriterPtr _file_writer;
+ std::unique_ptr<iceberg::Schema> _position_delete_schema;
std::unique_ptr<VFileFormatTransformer> _file_format_transformer;
int32_t _partition_spec_id = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
