This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2faf819fa89 [test](iceberg) Add partition evolution schema coverage
(#65870)
2faf819fa89 is described below
commit 2faf819fa895205215d9e0451c07563ddab2197b
Author: Gabriel <[email protected]>
AuthorDate: Tue Jul 28 10:49:34 2026 +0800
[test](iceberg) Add partition evolution schema coverage (#65870)
### What problem does this PR solve?
Related PR: #65502
Problem Summary:
#65502 already fixed the Iceberg scan correctness issue by sending the
complete current table schema to readers. Although that change was
introduced for equality-delete dependencies, the same all-column schema
invariant is also required for partition evolution: a column classified
as an identity partition column by a newer partition spec can still be
stored physically in files written by an older spec.
This follow-up does not change scan behavior. It documents the
partition-evolution invariant, extracts the existing schema
initialization into a testable helper, and adds a focused FE unit test
verifying that a non-projected evolved partition column remains in the
reader schema.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../datasource/iceberg/source/IcebergScanNode.java | 7 ++++-
.../iceberg/source/IcebergScanNodeTest.java | 33 ++++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 08c476718f6..5f5711679b6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -508,11 +508,16 @@ public class IcebergScanNode extends FileQueryScanNode {
super.createScanRangeLocations();
enableCurrentIcebergScanSemantics();
// Extract name mapping from Iceberg table properties
- Optional<Map<Integer, List<String>>> nameMapping =
extractNameMapping();
+ initializeIcebergSchemaInfo(extractNameMapping());
+ }
+ @VisibleForTesting
+ void initializeIcebergSchemaInfo(Optional<Map<Integer, List<String>>>
nameMapping) throws UserException {
// Equality-delete keys are hidden scan dependencies and need not
appear in the query
// projection. Both scanners need the complete current schema to
resolve field ids,
// historical names, types, and initial defaults when an old data file
lacks such a key.
+ // An identity partition column can also be a physical field in files
written by an older
+ // partition spec, so preserving the complete schema is required for
partition evolution.
ExternalUtil.initSchemaInfoForAllColumn(params, -1L,
source.getTargetTable().getColumns(),
nameMapping.orElse(Collections.emptyMap()),
nameMapping.isPresent(),
getBase64EncodedInitialDefaultsForScan());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
index 06d7b68c50d..0d5a00ade81 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
@@ -138,6 +138,12 @@ public class IcebergScanNodeTest {
enableCurrentIcebergScanSemantics();
return params.getIcebergScanSemanticsVersion();
}
+
+ TFileScanRangeParams initializeAndGetIcebergSchemaInfo() throws
UserException {
+ params = new TFileScanRangeParams();
+ initializeIcebergSchemaInfo(Optional.empty());
+ return params;
+ }
}
@Test
@@ -148,6 +154,33 @@ public class IcebergScanNodeTest {
node.enableAndGetIcebergScanSemanticsVersion());
}
+ @Test
+ public void testPartitionEvolutionKeepsNonFileSlotInReaderSchema() throws
Exception {
+ Column evolvedIdentityColumn = new Column("int_col", Type.BIGINT,
true);
+ evolvedIdentityColumn.setUniqueId(1);
+ Column projectedColumn = new Column("payload", Type.STRING, true);
+ projectedColumn.setUniqueId(2);
+
+ IcebergExternalTable targetTable =
Mockito.mock(IcebergExternalTable.class);
+ Mockito.when(targetTable.getColumns()).thenReturn(
+ List.of(evolvedIdentityColumn, projectedColumn));
+ IcebergSource source = Mockito.mock(IcebergSource.class);
+ Mockito.when(source.getTargetTable()).thenReturn(targetTable);
+
+ TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new
SessionVariable()));
+ node.addSlot(1, projectedColumn);
+ setIcebergSource(node, source);
+
Mockito.doReturn(Collections.emptyMap()).when(node).getBase64EncodedInitialDefaultsForScan();
+
+ TFileScanRangeParams scanParams =
node.initializeAndGetIcebergSchemaInfo();
+
+ Assert.assertEquals(2,
scanParams.getHistorySchemaInfo().get(0).getRootField().getFieldsSize());
+ Assert.assertEquals("int_col",
scanParams.getHistorySchemaInfo().get(0).getRootField()
+ .getFields().get(0).getFieldPtr().getName());
+ Assert.assertEquals("payload",
scanParams.getHistorySchemaInfo().get(0).getRootField()
+ .getFields().get(1).getFieldPtr().getName());
+ }
+
@Test
public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws
Exception {
TestIcebergScanNode node = new TestIcebergScanNode(new
SessionVariable());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
