This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ac15da94a24 [fix](fe) Load Iceberg views without engine-name metadata
(#69829)
ac15da94a24 is described below
commit ac15da94a247275b337f68a803a7354f0d7ce8f4
Author: daidai <[email protected]>
AuthorDate: Mon Nov 2 12:10:35 2026 +0800
[fix](fe) Load Iceberg views without engine-name metadata (#69829)
### What problem does this PR solve?
Allow missing/empty `engine-name` in Iceberg views. Preserve existing
engine-based SQL selection and only use the Doris/SDK fallback when the
field has no value.
### Release note
Support Iceberg views without `engine-name` while retaining existing SQL
selection when it is present.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [x] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [ ] No.
- [x] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../iceberg/IcebergConnectorMetadata.java | 27 ++---
.../iceberg/IcebergConnectorMetadataTest.java | 128 ++++++++++++++++++---
2 files changed, 121 insertions(+), 34 deletions(-)
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
index b571662d2b0..006f3b6773a 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
@@ -384,15 +384,8 @@ public class IcebergConnectorMetadata implements
ConnectorMetadata {
@Override
public ConnectorViewDefinition getViewDefinition(ConnectorSession session,
String dbName, String viewName) {
- // Mirror viewExists: wrap the remote load in the auth context and
normalize EVERY failure into a
- // RuntimeException (the seam's loadView already fails loud on a
non-view catalog with a
- // DorisConnectorException, which is a RuntimeException and surfaces
wrapped here). ONE remote load
- // yields both the sql/dialect (mirroring legacy
IcebergExternalTable.getViewText + getSqlDialect: the
- // dialect is the view-version summary's "engine-name", the SQL is
that dialect's representation) AND
- // the columns (parseSchema(view.schema()), mirroring legacy
IcebergUtils.loadViewSchemaCacheValue — a
- // view has NO partition columns and NO row-lineage). The
sql/dialect/column extraction lives HERE,
- // not in the SDK-only seam, because parseSchema reads the
enable.mapping.* flags that only exist in
- // this layer's properties (mirrors the table path: seam loadTable ->
metadata buildTableSchema).
+ // Resolve SQL and columns in one authenticated view load. Column
conversion belongs here because
+ // it uses this catalog's type-mapping flags. A view has no partition
or row-lineage columns.
try {
return executeAuthenticated(() -> {
View icebergView = catalogOps.loadView(dbName, viewName);
@@ -405,17 +398,17 @@ public class IcebergConnectorMetadata implements
ConnectorMetadata {
if (summary == null) {
throw new DorisConnectorException(String.format("Cannot
get summary for view '%s'", icebergView));
}
- // "engine-name" is the iceberg view-version summary key the
writing engine (e.g. spark) records.
String engineName = summary.get("engine-name");
- if (engineName == null || engineName.isEmpty()) {
- throw new DorisConnectorException(
- String.format("Cannot get engine-name for view
'%s'", icebergView));
- }
- String dialect = engineName.toLowerCase(Locale.ROOT);
- SQLViewRepresentation sqlViewRepresentation =
icebergView.sqlFor(dialect);
+ // Preserve engine-based SQL selection for existing views.
Only a missing/empty engine-name
+ // uses Doris as the preference; Iceberg falls back to the
first SQL if it has no match.
+ String preferredDialect = engineName == null ||
engineName.isEmpty()
+ ? "doris" : engineName.toLowerCase(Locale.ROOT);
+ SQLViewRepresentation sqlViewRepresentation =
icebergView.sqlFor(preferredDialect);
if (sqlViewRepresentation == null) {
- throw new DorisConnectorException("Cannot get view text
from iceberg view");
+ throw new DorisConnectorException(String.format(
+ "Cannot get SQL representation for Iceberg view
'%s.%s'", dbName, viewName));
}
+ String dialect =
sqlViewRepresentation.dialect().toLowerCase(Locale.ROOT);
List<ConnectorColumn> columns =
parseSchema(icebergView.schema());
return new
ConnectorViewDefinition(sqlViewRepresentation.sql(), dialect, columns);
});
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
index 877ecfb65e1..ccce63017af 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
@@ -41,8 +41,12 @@ import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.inmemory.InMemoryCatalog;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.view.BaseView;
import org.apache.iceberg.view.ImmutableSQLViewRepresentation;
import org.apache.iceberg.view.ImmutableViewVersion;
+import org.apache.iceberg.view.View;
+import org.apache.iceberg.view.ViewMetadata;
+import org.apache.iceberg.view.ViewOperations;
import org.apache.iceberg.view.ViewVersion;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -144,6 +148,29 @@ public class IcebergConnectorMetadataTest {
return builder.build();
}
+ private static View viewWithMetadata(ViewVersion version, Schema schema) {
+ ViewMetadata metadata = ViewMetadata.builder()
+ .setLocation("s3://warehouse/db1/v1")
+ .setCurrentVersion(version, schema)
+ .build();
+ return new BaseView(new ViewOperations() {
+ @Override
+ public ViewMetadata current() {
+ return metadata;
+ }
+
+ @Override
+ public ViewMetadata refresh() {
+ return metadata;
+ }
+
+ @Override
+ public void commit(ViewMetadata base, ViewMetadata updated) {
+ throw new UnsupportedOperationException();
+ }
+ }, "db1.v1");
+ }
+
// ---------------------------------------------------------------------
// write capabilities (row-level DML dispatch)
// ---------------------------------------------------------------------
@@ -612,15 +639,14 @@ public class IcebergConnectorMetadataTest {
Schema viewSchema = new Schema(
Types.NestedField.required(1, "id", Types.IntegerType.get()),
Types.NestedField.optional(2, "name", Types.StringType.get()));
- ops.view = new FakeIcebergViewCatalog.StubView(
+ ops.view = viewWithMetadata(
viewVersionWith("Spark", "spark", "SELECT 1"), viewSchema);
ConnectorViewDefinition def =
metadataWith(ops).getViewDefinition(null, "db1", "v1");
// WHY (H8): ONE loadView yields the sql/dialect (BindRelation / SHOW
CREATE) AND the column schema
- // (DESC / SHOW COLUMNS / information_schema.columns). The dialect is
the summary engine-name LOWERCASED;
- // the sql is that dialect's representation; the columns are
parseSchema(view.schema()). MUTATION:
- // dropping toLowerCase / wrong representation / not parsing the view
schema (empty columns) -> red.
+ // (DESC / SHOW COLUMNS / information_schema.columns). SQL and dialect
come from the selected
+ // representation, while columns come from parseSchema(view.schema()).
Assertions.assertEquals("SELECT 1", def.getSql());
Assertions.assertEquals("spark", def.getDialect());
List<ConnectorColumn> cols = def.getColumns();
@@ -644,7 +670,7 @@ public class IcebergConnectorMetadataTest {
Schema viewSchema = new Schema(
Types.NestedField.optional(1, "b", Types.BinaryType.get()),
Types.NestedField.optional(2, "ts_tz",
Types.TimestampType.withZone()));
- ops.view = new FakeIcebergViewCatalog.StubView(
+ ops.view = viewWithMetadata(
viewVersionWith("spark", "spark", "SELECT 1"), viewSchema);
Map<String, String> props = new HashMap<>();
props.put("enable.mapping.varbinary", "true");
@@ -690,26 +716,94 @@ public class IcebergConnectorMetadataTest {
}
@Test
- public void getViewDefinitionThrowsWhenEngineNameMissing() {
- // WHY (moved from the seam test post-H8): the dialect IS the summary
engine-name; without it the SQL
- // representation cannot be selected. MUTATION: dropping the
empty-engine-name check -> wrong behavior.
+ public void getViewDefinitionDoesNotRequireEngineName() {
+ for (String engineName : Arrays.asList(null, "", "Polaris")) {
+ RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
+ ops.view = viewWithMetadata(viewVersionWith(engineName, "SpArK",
"SELECT 1"), idNameSchema());
+ ops.throwNoSuchTableOnLoadTable = true;
+
+ ConnectorViewDefinition definition =
metadataWith(ops).getViewDefinition(null, "db1", "v1");
+
+ Assertions.assertEquals("SELECT 1", definition.getSql());
+ Assertions.assertEquals("spark", definition.getDialect());
+ Assertions.assertEquals(2, definition.getColumns().size());
+
Assertions.assertEquals(Collections.singletonList("loadView:db1.v1"), ops.log);
+ }
+ }
+
+ @Test
+ public void getViewDefinitionPrefersDorisWhenEngineNameIsMissing() {
RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
- ops.view = new FakeIcebergViewCatalog.StubView(
- viewVersionWith(null, "spark", "SELECT 1"), idNameSchema());
- Assertions.assertThrows(RuntimeException.class,
- () -> metadataWith(ops).getViewDefinition(null, "db1", "v1"));
+ ViewVersion version = ImmutableViewVersion.builder()
+ .from(viewVersionWith(null, "spark", "SELECT 1 AS
spark_value"))
+ .addRepresentations(ImmutableSQLViewRepresentation.builder()
+ .sql("SELECT 1 AS
doris_value").dialect("DoRiS").build())
+ .build();
+ ops.view = viewWithMetadata(version, idNameSchema());
+
+ ConnectorViewDefinition definition =
metadataWith(ops).getViewDefinition(null, "db1", "v1");
+
+ Assertions.assertEquals("SELECT 1 AS doris_value",
definition.getSql());
+ Assertions.assertEquals("doris", definition.getDialect());
}
@Test
- public void getViewDefinitionThrowsWhenNoSqlForDialect() {
- // WHY (moved from the seam test post-H8): engine-name present but no
SQL representation for that dialect
- // -> sqlFor returns null -> must fail loud (mirrors legacy "Cannot
get view text"). MUTATION: dropping
- // the null-sql check -> NPE.
+ public void getViewDefinitionKeepsEngineDialectWhenDorisIsAbsent() {
+ RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
+ ViewVersion version = ImmutableViewVersion.builder()
+ .from(viewVersionWith("Trino", "spark", "SELECT 1 AS
spark_value"))
+ .addRepresentations(ImmutableSQLViewRepresentation.builder()
+ .sql("SELECT 1 AS
trino_value").dialect("trino").build())
+ .build();
+ ops.view = viewWithMetadata(version, idNameSchema());
+
+ ConnectorViewDefinition definition =
metadataWith(ops).getViewDefinition(null, "db1", "v1");
+
+ Assertions.assertEquals("SELECT 1 AS trino_value",
definition.getSql());
+ Assertions.assertEquals("trino", definition.getDialect());
+ }
+
+ @Test
+ public void getViewDefinitionKeepsEngineDialectWhenDorisIsAvailable() {
+ RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
+ ViewVersion version = ImmutableViewVersion.builder()
+ .from(viewVersionWith("Spark", "spark", "SELECT 1 AS
spark_value"))
+ .addRepresentations(ImmutableSQLViewRepresentation.builder()
+ .sql("SELECT 1 AS
doris_value").dialect("doris").build())
+ .build();
+ ops.view = viewWithMetadata(version, idNameSchema());
+
+ ConnectorViewDefinition definition =
metadataWith(ops).getViewDefinition(null, "db1", "v1");
+
+ Assertions.assertEquals("SELECT 1 AS spark_value",
definition.getSql());
+ Assertions.assertEquals("spark", definition.getDialect());
+ }
+
+ @Test
+ public void getViewDefinitionUsesFirstSqlWhenEngineNameAndDorisAreAbsent()
{
+ RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
+ ViewVersion version = ImmutableViewVersion.builder()
+ .from(viewVersionWith(null, "spark", "SELECT 1 AS
spark_value"))
+ .addRepresentations(ImmutableSQLViewRepresentation.builder()
+ .sql("SELECT 1 AS
trino_value").dialect("trino").build())
+ .build();
+ ops.view = viewWithMetadata(version, idNameSchema());
+
+ ConnectorViewDefinition definition =
metadataWith(ops).getViewDefinition(null, "db1", "v1");
+
+ Assertions.assertEquals("SELECT 1 AS spark_value",
definition.getSql());
+ Assertions.assertEquals("spark", definition.getDialect());
+ }
+
+ @Test
+ public void getViewDefinitionThrowsWhenNoSqlRepresentation() {
RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
ops.view = new FakeIcebergViewCatalog.StubView(
viewVersionWith("spark", null, null), idNameSchema());
- Assertions.assertThrows(RuntimeException.class,
+ RuntimeException exception =
Assertions.assertThrows(RuntimeException.class,
() -> metadataWith(ops).getViewDefinition(null, "db1", "v1"));
+ Assertions.assertTrue(exception.getMessage().contains(
+ "Cannot get SQL representation for Iceberg view 'db1.v1'"),
exception.getMessage());
}
// ---------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
