This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d96c5fdcbc6 branch-3.1 [feature](iceberg catalog) support iceberg view
query #51376 (#52806)
d96c5fdcbc6 is described below
commit d96c5fdcbc60e0657d2a7b88ffb8e38c7edbd362
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Jul 7 13:50:16 2025 +0800
branch-3.1 [feature](iceberg catalog) support iceberg view query #51376
(#52806)
bp #51376
---------
Co-authored-by: heguanhui <[email protected]>
---
.../create_preinstalled_scripts/iceberg/run14.sql | 87 +++++++++
.../main/java/org/apache/doris/common/Config.java | 3 +
.../main/java/org/apache/doris/catalog/Env.java | 6 +-
.../apache/doris/datasource/ExternalCatalog.java | 11 ++
.../doris/datasource/hive/HMSExternalTable.java | 4 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 12 +-
.../datasource/iceberg/IcebergExternalTable.java | 108 ++++++++++-
.../datasource/iceberg/IcebergMetadataCache.java | 41 +++++
.../datasource/iceberg/IcebergMetadataOps.java | 106 ++++++++++-
.../doris/datasource/iceberg/IcebergUtils.java | 81 +++++---
.../iceberg/source/IcebergApiSource.java | 11 ++
.../datasource/operations/ExternalMetadataOps.java | 33 ++++
.../doris/nereids/rules/analysis/BindRelation.java | 43 ++++-
.../org/apache/doris/planner/IcebergTableSink.java | 3 +
.../java/org/apache/doris/qe/ShowExecutor.java | 8 +
.../iceberg/test_iceberg_view_query_p0.out | Bin 0 -> 1033 bytes
.../iceberg/test_iceberg_view_query_p0.groovy | 203 +++++++++++++++++++++
17 files changed, 712 insertions(+), 48 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run14.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run14.sql
new file mode 100644
index 00000000000..da4f0035c65
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run14.sql
@@ -0,0 +1,87 @@
+use demo.test_db;
+drop table if exists test_db.t_partitioned_table;
+drop table if exists test_db.t_unpartitioned_table;
+drop table if exists test_db.t_product_info;
+drop table if exists test_db.t_sales_record;
+CREATE TABLE IF NOT EXISTS t_partitioned_table (
+ col1 int,
+ col2 string,
+ col3 int,
+ col4 string,
+ col5 string
+) USING iceberg
+PARTITIONED BY (col5);
+CREATE TABLE IF NOT EXISTS t_unpartitioned_table (
+ col1 INT,
+ col2 VARCHAR(100),
+ col3 DECIMAL(10, 2)
+) USING iceberg;
+CREATE TABLE IF NOT EXISTS t_product_info (
+ product_id INT,
+ product_name VARCHAR(100),
+ price DECIMAL(10, 2)
+) USING iceberg;
+CREATE TABLE IF NOT EXISTS t_sales_record (
+ sales_id INT,
+ order_id INT,
+ product_id INT,
+ quantity_sold INT,
+ sale_date DATE
+) USING iceberg;
+INSERT INTO t_partitioned_table (col1, col2, col3, col4, col5)
+VALUES
+(1, 'Alice', 25, 'Female', 'New York'),
+(2, 'Bob', 30, 'Male', 'Los Angeles'),
+(3, 'Charlie', 35, 'Male', 'Chicago'),
+(4, 'David', 22, 'Male', 'Houston'),
+(5, 'Eve', 28, 'Female', 'Phoenix');
+INSERT INTO t_unpartitioned_table (col1, col2, col3)
+VALUES
+(1001, 'Product A', 20.00),
+(1002, 'Product B', 30.00),
+(1003, 'Product C', 40.00),
+(1004, 'Product D', 50.00),
+(1005, 'Product E', 60.00);
+INSERT INTO t_product_info (product_id, product_name, price)
+VALUES
+(1001, 'Product A', 20.00),
+(1002, 'Product B', 30.00),
+(1003, 'Product C', 40.00),
+(1004, 'Product D', 50.00),
+(1005, 'Product E', 60.00);
+INSERT INTO t_sales_record (sales_id, order_id, product_id, quantity_sold,
sale_date)
+VALUES
+(7001, 101, 1001, 2, date '2024-01-01'),
+(7002, 102, 1002, 3, date '2024-01-02'),
+(7003, 103, 1003, 1, date '2024-01-03'),
+(7004, 104, 1001, 1, date '2024-01-04'),
+(7005, 105, 1004, 4, date '2024-01-05');
+create view v_with_unpartitioned_table
+as
+select
+ *
+from
+ t_unpartitioned_table;
+create view v_with_partitioned_table
+ as
+ select * from t_partitioned_table order by col1;
+create view v_with_partitioned_column
+as
+select
+ col5
+from
+ t_partitioned_table;
+create view v_with_joint_table
+as
+SELECT
+ pi.product_name,
+ sr.quantity_sold,
+ sr.sale_date
+FROM
+ t_product_info pi
+ JOIN
+ t_sales_record sr
+ ON pi.product_id = sr.product_id
+WHERE
+ sr.sale_date BETWEEN '2024-01-01' AND '2024-01-03'
+ AND sr.quantity_sold > 1;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index cb41a970ace..5fcb5165d38 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2266,6 +2266,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean enable_query_hive_views = true;
+ @ConfField(mutable = true)
+ public static boolean enable_query_iceberg_views = true;
+
/**
* If set to true, doris will automatically synchronize hms metadata to
the cache in fe.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index a6f150d1949..2e91f173007 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4190,10 +4190,10 @@ public class Env {
sb.append("\n)");
} else if (table.getType() == TableType.ICEBERG_EXTERNAL_TABLE) {
addTableComment(table, sb);
- org.apache.iceberg.Table icebergTable = ((IcebergExternalTable)
table).getIcebergTable();
- sb.append("\nLOCATION
'").append(icebergTable.location()).append("'");
+ IcebergExternalTable icebergExternalTable = (IcebergExternalTable)
table;
+ sb.append("\nLOCATION
'").append(icebergExternalTable.location()).append("'");
sb.append("\nPROPERTIES (");
- Iterator<Entry<String, String>> iterator =
icebergTable.properties().entrySet().iterator();
+ Iterator<Entry<String, String>> iterator =
icebergExternalTable.properties().entrySet().iterator();
while (iterator.hasNext()) {
Entry<String, String> prop = iterator.next();
sb.append("\n \"").append(prop.getKey()).append("\" =
\"").append(prop.getValue()).append("\"");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 29287dc57b6..bed32c62b85 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -1282,4 +1282,15 @@ public abstract class ExternalCatalog
public ThreadPoolExecutor getThreadPoolExecutor() {
return threadPoolWithPreAuth;
}
+
+ /**
+ * Check if an external view exists.
+ * @param dbName
+ * @param viewName
+ * @return
+ */
+ public boolean viewExists(String dbName, String viewName) {
+ throw new UnsupportedOperationException("View is not supported.");
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 4504e2a1f34..f944c704abe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -637,7 +637,8 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
private Optional<SchemaCacheValue> getIcebergSchema(SchemaCacheKey key) {
- return IcebergUtils.loadSchemaCacheValue(catalog, dbName, name,
((IcebergSchemaCacheKey) key).getSchemaId());
+ return IcebergUtils.loadSchemaCacheValue(
+ catalog, dbName, name, ((IcebergSchemaCacheKey)
key).getSchemaId(), isView());
}
private Optional<SchemaCacheValue> getHudiSchema(SchemaCacheKey key) {
@@ -1143,4 +1144,5 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
makeSureInitialized();
return dlaTable.isValidRelatedTable();
}
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index 41149370097..4aeb96f6e37 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -106,7 +106,12 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- return metadataOps.listTableNames(dbName);
+ // On the Doris side, the result of SHOW TABLES for Iceberg external
tables includes both tables and views,
+ // so the combined set of tables and views is used here.
+ List<String> tableNames = metadataOps.listTableNames(dbName);
+ List<String> viewNames = metadataOps.listViewNames(dbName);
+ tableNames.addAll(viewNames);
+ return tableNames;
}
@Override
@@ -121,4 +126,9 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
Map<String, String> properties = catalogProperty.getHadoopProperties();
conf.set(Constants.AWS_CREDENTIALS_PROVIDER,
PropertyConverter.getAWSCredentialsProviders(properties));
}
+
+ @Override
+ public boolean viewExists(String dbName, String viewName) {
+ return metadataOps.viewExists(dbName, viewName);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
index 0813474b45f..bb947385f3b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
@@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.datasource.mvcc.EmptyMvccSnapshot;
import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.datasource.mvcc.MvccTable;
import org.apache.doris.datasource.systable.SupportedSysTables;
@@ -48,9 +49,13 @@ import org.apache.doris.thrift.TTableType;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.PartitionField;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Table;
+import org.apache.iceberg.view.SQLViewRepresentation;
+import org.apache.iceberg.view.View;
+import org.apache.iceberg.view.ViewVersion;
import java.util.HashMap;
import java.util.List;
@@ -64,6 +69,8 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
private Table table;
private boolean isValidRelatedTableCached = false;
private boolean isValidRelatedTable = false;
+ private boolean isView;
+ private static final String ENGINE_PROP_NAME = "engine-name";
public IcebergExternalTable(long id, String name, String remoteName,
IcebergExternalCatalog catalog,
IcebergExternalDatabase db) {
@@ -78,6 +85,7 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
super.makeSureInitialized();
if (!objectCreated) {
objectCreated = true;
+ isView = catalog.viewExists(dbName, getRemoteName());
}
}
@@ -88,7 +96,10 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
@Override
public Optional<SchemaCacheValue> initSchema(SchemaCacheKey key) {
- return IcebergUtils.loadSchemaCacheValue(catalog, dbName, name,
((IcebergSchemaCacheKey) key).getSchemaId());
+ boolean isView = isView();
+ String tableName = getRemoteName();
+ return IcebergUtils.loadSchemaCacheValue(
+ catalog, dbName, tableName, ((IcebergSchemaCacheKey)
key).getSchemaId(), isView);
}
@Override
@@ -230,8 +241,12 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
@Override
public MvccSnapshot loadSnapshot(Optional<TableSnapshot> tableSnapshot,
Optional<TableScanParams> scanParams) {
- return new
IcebergMvccSnapshot(IcebergUtils.getIcebergSnapshotCacheValue(
+ if (isView()) {
+ return new EmptyMvccSnapshot();
+ } else {
+ return new
IcebergMvccSnapshot(IcebergUtils.getIcebergSnapshotCacheValue(
tableSnapshot, getCatalog(), getDbName(), getName(),
scanParams));
+ }
}
@Override
@@ -263,4 +278,93 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
makeSureInitialized();
return SupportedSysTables.ICEBERG_SUPPORTED_SYS_TABLES;
}
+
+ @Override
+ public boolean isView() {
+ makeSureInitialized();
+ return isView;
+ }
+
+ public String getViewText() {
+ try {
+ return catalog.getPreExecutionAuthenticator().execute(() -> {
+ View icebergView = IcebergUtils.getIcebergView(getCatalog(),
dbName, getRemoteName());
+ ViewVersion viewVersion = icebergView.currentVersion();
+ if (viewVersion == null) {
+ throw new RuntimeException(String.format("Cannot get view
version for view '%s'", icebergView));
+ }
+ Map<String, String> summary = viewVersion.summary();
+ if (summary == null) {
+ throw new RuntimeException(String.format("Cannot get
summary for view '%s'", icebergView));
+ }
+ String engineName = summary.get(ENGINE_PROP_NAME);
+ if (StringUtils.isEmpty(engineName)) {
+ throw new RuntimeException(String.format("Cannot get
engine-name for view '%s'", icebergView));
+ }
+ SQLViewRepresentation sqlViewRepresentation =
icebergView.sqlFor(engineName.toLowerCase());
+ if (sqlViewRepresentation == null) {
+ throw new UnsupportedOperationException("Cannot get view
text from iceberg view");
+ }
+ return sqlViewRepresentation.sql();
+ });
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getSqlDialect() {
+ try {
+ return catalog.getPreExecutionAuthenticator().execute(() -> {
+ View icebergView = IcebergUtils.getIcebergView(getCatalog(),
dbName, getRemoteName());
+ ViewVersion viewVersion = icebergView.currentVersion();
+ if (viewVersion == null) {
+ throw new RuntimeException(String.format("Cannot get view
version for view '%s'", icebergView));
+ }
+ Map<String, String> summary = viewVersion.summary();
+ if (summary == null) {
+ throw new RuntimeException(String.format("Cannot get
summary for view '%s'", icebergView));
+ }
+ String engineName = summary.get(ENGINE_PROP_NAME);
+ if (StringUtils.isEmpty(engineName)) {
+ throw new RuntimeException(String.format("Cannot get
engine-name for view '%s'", icebergView));
+ }
+ return engineName.toLowerCase();
+ });
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public View getIcebergView() {
+ return IcebergUtils.getIcebergView(getCatalog(), dbName,
getRemoteName());
+ }
+
+ /**
+ * get location of an iceberg table or view
+ * @return
+ */
+ public String location() {
+ if (isView()) {
+ View icebergView = getIcebergView();
+ return icebergView.location();
+ } else {
+ Table icebergTable = getIcebergTable();
+ return icebergTable.location();
+ }
+ }
+
+ /**
+ * get properties of an iceberg table or view
+ * @return
+ */
+ public Map<String, String> properties() {
+ if (isView()) {
+ View icebergView = getIcebergView();
+ return icebergView.properties();
+ } else {
+ Table icebergTable = getIcebergTable();
+ return icebergTable.properties();
+ }
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
index e41fa620b9c..67cf1d3855a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
@@ -35,6 +35,7 @@ import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.SerializableTable;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
+import org.apache.iceberg.view.View;
import org.jetbrains.annotations.NotNull;
import java.util.HashMap;
@@ -49,6 +50,7 @@ public class IcebergMetadataCache {
private final LoadingCache<IcebergMetadataCacheKey, List<Snapshot>>
snapshotListCache;
private final LoadingCache<IcebergMetadataCacheKey, Table> tableCache;
private final LoadingCache<IcebergMetadataCacheKey,
IcebergSnapshotCacheValue> snapshotCache;
+ private final LoadingCache<IcebergMetadataCacheKey, View> viewCache;
public IcebergMetadataCache(ExecutorService executor) {
CacheFactory snapshotListCacheFactory = new CacheFactory(
@@ -74,6 +76,7 @@ public class IcebergMetadataCache {
true,
null);
this.snapshotCache = snapshotCacheFactory.buildCache(key ->
loadSnapshot(key), null, executor);
+ this.viewCache = tableCacheFactory.buildCache(key -> loadView(key),
null, executor);
}
public Table getIcebergTable(CatalogIf catalog, String dbName, String
tbName) {
@@ -153,6 +156,10 @@ public class IcebergMetadataCache {
snapshotCache.asMap().keySet().stream()
.filter(key -> key.catalog.getId() == catalogId)
.forEach(snapshotCache::invalidate);
+
+ viewCache.asMap().entrySet().stream()
+ .filter(entry -> entry.getKey().catalog.getId() == catalogId)
+ .forEach(entry -> viewCache.invalidate(entry.getKey()));
}
public void invalidateTableCache(long catalogId, String dbName, String
tblName) {
@@ -176,6 +183,13 @@ public class IcebergMetadataCache {
.filter(key -> key.catalog.getId() == catalogId &&
key.dbName.equals(dbName) && key.tableName.equals(
tblName))
.forEach(snapshotCache::invalidate);
+ viewCache.asMap().entrySet().stream()
+ .filter(entry -> {
+ IcebergMetadataCacheKey key = entry.getKey();
+ return key.catalog.getId() == catalogId &&
key.dbName.equals(dbName) && key.tableName.equals(
+ tblName);
+ })
+ .forEach(entry -> viewCache.invalidate(entry.getKey()));
}
public void invalidateDbCache(long catalogId, String dbName) {
@@ -196,6 +210,12 @@ public class IcebergMetadataCache {
snapshotCache.asMap().keySet().stream()
.filter(key -> key.catalog.getId() == catalogId &&
key.dbName.equals(dbName))
.forEach(snapshotCache::invalidate);
+ viewCache.asMap().entrySet().stream()
+ .filter(entry -> {
+ IcebergMetadataCacheKey key = entry.getKey();
+ return key.catalog.getId() == catalogId &&
key.dbName.equals(dbName);
+ })
+ .forEach(entry -> viewCache.invalidate(entry.getKey()));
}
private static void initIcebergTableFileIO(Table table, Map<String,
String> props) {
@@ -257,4 +277,25 @@ public class IcebergMetadataCache {
snapshotCache.estimatedSize()));
return res;
}
+
+ private View loadView(IcebergMetadataCacheKey key) {
+ IcebergMetadataOps ops;
+ if (key.catalog instanceof IcebergExternalCatalog) {
+ ops = (IcebergMetadataOps) (((IcebergExternalCatalog)
key.catalog).getMetadataOps());
+ } else {
+ return null;
+ }
+ try {
+ return ((ExternalCatalog)
key.catalog).getPreExecutionAuthenticator().execute(() ->
+ ops.loadView(key.dbName, key.tableName));
+ } catch (Exception e) {
+ throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e),
e);
+ }
+
+ }
+
+ public View getIcebergView(CatalogIf catalog, String dbName, String
tbName) {
+ IcebergMetadataCacheKey key = IcebergMetadataCacheKey.of(catalog,
dbName, tbName);
+ return viewCache.get(key);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 9f304927b66..60e8d4343f5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -42,10 +42,14 @@ import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.view.View;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -127,7 +131,24 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
try {
return preExecutionAuthenticator.execute(() -> {
List<TableIdentifier> tableIdentifiers =
catalog.listTables(getNamespace(dbName));
- return
tableIdentifiers.stream().map(TableIdentifier::name).collect(Collectors.toList());
+ List<String> views;
+ // Our original intention was simply to clearly define the
responsibilities of ViewCatalog and Catalog.
+ // IcebergMetadataOps handles listTableNames and listViewNames
separately.
+ // listTableNames should only focus on the table type,
+ // but in reality, Iceberg's return includes views. Therefore,
we added a filter to exclude views.
+ if (catalog instanceof ViewCatalog) {
+ views = ((ViewCatalog)
catalog).listViews(getNamespace(dbName))
+
.stream().map(TableIdentifier::name).collect(Collectors.toList());
+ } else {
+ views = Collections.emptyList();
+ }
+ if (views.isEmpty()) {
+ return
tableIdentifiers.stream().map(TableIdentifier::name).collect(Collectors.toList());
+ } else {
+ return tableIdentifiers.stream()
+ .map(TableIdentifier::name)
+ .filter(name ->
!views.contains(name)).collect(Collectors.toList());
+ }
});
} catch (Exception e) {
throw new RuntimeException("Failed to list table names, error
message is:" + e.getMessage(), e);
@@ -246,7 +267,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
.map(col -> new StructField(col.getName(), col.getType(),
col.getComment(), col.isAllowNull()))
.collect(Collectors.toList());
StructType structType = new StructType(new ArrayList<>(collect));
- org.apache.iceberg.types.Type visit =
+ Type visit =
DorisTypeVisitor.visit(structType, new
DorisTypeToIcebergType(structType));
Schema schema = new
Schema(visit.asNestedType().asStructType().fields());
Map<String, String> properties = stmt.getProperties();
@@ -268,7 +289,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
public void dropTableImpl(DropTableStmt stmt) throws DdlException {
try {
preExecutionAuthenticator.execute(() -> {
- performDropTable(stmt);
+ if
(getExternalCatalog().getMetadataOps().viewExists(stmt.getDbName(),
stmt.getTableName())) {
+ performDropView(stmt.getDbName(), stmt.getTableName(),
stmt.isSetIfExists());
+ } else {
+ performDropTable(stmt.getDbName(), stmt.getTableName(),
stmt.isSetIfExists());
+ }
return null;
});
} catch (Exception e) {
@@ -285,13 +310,6 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
- private void performDropTable(DropTableStmt stmt) throws DdlException {
- if (stmt == null) {
- throw new DdlException("DropTableStmt is null");
- }
- performDropTable(stmt.getDbName(), stmt.getTableName(),
stmt.isSetIfExists());
- }
-
private void performDropTable(String dbName, String tableName, boolean
ifExists) throws DdlException {
ExternalDatabase<?> db = dorisCatalog.getDbNullable(dbName);
if (db == null) {
@@ -332,6 +350,47 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
+ @Override
+ public boolean viewExists(String dbName, String viewName) {
+ if (!(catalog instanceof ViewCatalog)) {
+ return false;
+ }
+ try {
+ return preExecutionAuthenticator.execute(() ->
+ ((ViewCatalog)
catalog).viewExists(getTableIdentifier(dbName, viewName)));
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to check view exist, error
message is:" + e.getMessage(), e);
+
+ }
+ }
+
+ @Override
+ public View loadView(String dbName, String tblName) {
+ if (!(catalog instanceof ViewCatalog)) {
+ return null;
+ }
+ try {
+ ViewCatalog viewCatalog = (ViewCatalog) catalog;
+ return preExecutionAuthenticator.execute(() ->
viewCatalog.loadView(TableIdentifier.of(dbName, tblName)));
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to load view, error message
is:" + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public List<String> listViewNames(String db) {
+ if (!(catalog instanceof ViewCatalog)) {
+ return Collections.emptyList();
+ }
+ try {
+ return preExecutionAuthenticator.execute(() ->
+ ((ViewCatalog) catalog).listViews(Namespace.of(db))
+
.stream().map(TableIdentifier::name).collect(Collectors.toList()));
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to list view names, error
message is:" + e.getMessage(), e);
+ }
+ }
+
private TableIdentifier getTableIdentifier(String dbName, String tblName) {
return externalCatalogName
.map(s -> TableIdentifier.of(s, dbName, tblName))
@@ -351,4 +410,31 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
public ThreadPoolExecutor getThreadPoolWithPreAuth() {
return dorisCatalog.getThreadPoolExecutor();
}
+
+ private void performDropView(String dbName, String viewName, boolean
ifExists) throws DdlException {
+ if (!(catalog instanceof ViewCatalog)) {
+ throw new DdlException("Drop Iceberg view is not supported with
not view catalog.");
+ }
+ ExternalDatabase<?> db = dorisCatalog.getDbNullable(dbName);
+ if (db == null) {
+ if (ifExists) {
+ LOG.info("database [{}] does not exist when drop view[{}]",
dbName, viewName);
+ return;
+ } else {
+ ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR,
dbName);
+ }
+ }
+ ViewCatalog viewCatalog = (ViewCatalog) catalog;
+ if (!viewExists(dbName, viewName)) {
+ if (ifExists) {
+ LOG.info("drop view[{}] which does not exist", viewName);
+ return;
+ } else {
+ ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TABLE,
viewName, dbName);
+ }
+ }
+ viewCatalog.dropView(getTableIdentifier(dbName, viewName));
+ db.setUnInitialized(true);
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index e956e236903..73d4a2fa76a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -103,6 +103,7 @@ import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.LocationUtil;
import org.apache.iceberg.util.SnapshotUtil;
import org.apache.iceberg.util.StructProjection;
+import org.apache.iceberg.view.View;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -629,18 +630,29 @@ public class IcebergUtils {
/**
* Get iceberg schema from catalog and convert them to doris schema
*/
- public static List<Column> getSchema(ExternalCatalog catalog, String
dbName, String name, long schemaId) {
+ public static List<Column> getSchema(
+ ExternalCatalog catalog, String dbName, String name, long
schemaId, boolean isView) {
try {
return catalog.getPreExecutionAuthenticator().execute(() -> {
- org.apache.iceberg.Table icebergTable =
getIcebergTable(catalog, dbName, name);
Schema schema;
- if (schemaId == NEWEST_SCHEMA_ID ||
icebergTable.currentSnapshot() == null) {
- schema = icebergTable.schema();
+ if (isView) {
+ View icebergView = getIcebergView(catalog, dbName, name);
+ if (schemaId == NEWEST_SCHEMA_ID) {
+ schema = icebergView.schema();
+ } else {
+ schema = icebergView.schemas().get((int) schemaId);
+ }
} else {
- schema = icebergTable.schemas().get((int) schemaId);
+ Table icebergTable = getIcebergTable(catalog, dbName,
name);
+ if (schemaId == NEWEST_SCHEMA_ID ||
icebergTable.currentSnapshot() == null) {
+ schema = icebergTable.schema();
+ } else {
+ schema = icebergTable.schemas().get((int) schemaId);
+ }
}
+ String type = isView ? "view" : "table";
Preconditions.checkNotNull(schema,
- "Schema for table " + catalog.getName() + "." + dbName
+ "." + name + " is null");
+ "Schema for " + type + " " + catalog.getName() + "." +
dbName + "." + name + " is null");
return parseSchema(schema);
});
} catch (Exception e) {
@@ -1169,25 +1181,6 @@ public class IcebergUtils {
}
}
- // load table schema from iceberg API to external schema cache.
- public static Optional<SchemaCacheValue> loadSchemaCacheValue(
- ExternalCatalog catalog, String dbName, String tbName, long
schemaId) {
- Table table = IcebergUtils.getIcebergTable(catalog, dbName, tbName);
- List<Column> schema = IcebergUtils.getSchema(catalog, dbName, tbName,
schemaId);
- List<Column> tmpColumns = Lists.newArrayList();
- PartitionSpec spec = table.spec();
- for (PartitionField field : spec.fields()) {
- Types.NestedField col = table.schema().findField(field.sourceId());
- for (Column c : schema) {
- if (c.getName().equalsIgnoreCase(col.name())) {
- tmpColumns.add(c);
- break;
- }
- }
- }
- return Optional.of(new IcebergSchemaCacheValue(schema, tmpColumns));
- }
-
public static List<Column> getIcebergSchema(
TableIf tableIf,
ExternalCatalog catalog,
@@ -1213,4 +1206,42 @@ public class IcebergUtils {
Optional.empty(), catalog, dbName, tbName, Optional.empty());
}
}
+
+ public static org.apache.iceberg.view.View getIcebergView(ExternalCatalog
catalog, String dbName, String tblName) {
+ return getIcebergViewInternal(catalog, dbName, tblName);
+ }
+
+ private static org.apache.iceberg.view.View
getIcebergViewInternal(ExternalCatalog catalog, String dbName,
+ String tblName) {
+ IcebergMetadataCache metadataCache =
Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache();
+ return metadataCache.getIcebergView(catalog, dbName, tblName);
+ }
+
+ public static Optional<SchemaCacheValue> loadSchemaCacheValue(
+ ExternalCatalog catalog, String dbName, String tbName, long
schemaId, boolean isView) {
+ List<Column> schema = IcebergUtils.getSchema(catalog, dbName, tbName,
schemaId, isView);
+ List<Column> tmpColumns = Lists.newArrayList();
+ if (!isView) {
+ // get table partition column info
+ Table table = IcebergUtils.getIcebergTable(catalog, dbName,
tbName);
+ PartitionSpec spec = table.spec();
+ for (PartitionField field : spec.fields()) {
+ Types.NestedField col =
table.schema().findField(field.sourceId());
+ for (Column c : schema) {
+ if (c.getName().equalsIgnoreCase(col.name())) {
+ tmpColumns.add(c);
+ break;
+ }
+ }
+ }
+ }
+ return Optional.of(new IcebergSchemaCacheValue(schema, tmpColumns));
+ }
+
+ public static String showCreateView(IcebergExternalTable
icebergExternalTable) {
+ return String.format("CREATE VIEW `%s` AS ",
icebergExternalTable.getName())
+ +
+ icebergExternalTable.getViewText();
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
index 4b4e76bea47..8ba1d71c1cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
@@ -42,6 +42,17 @@ public class IcebergApiSource implements IcebergSource {
public IcebergApiSource(IcebergExternalTable table, TupleDescriptor desc,
Map<String, ColumnRange> columnNameToRange) {
+ // Theoretically, the IcebergScanNode is responsible for scanning data
from physical tables.
+ // Views should not reach this point.
+ // By adding this validation, we aim to ensure that if a view query
does end up here, it indicates a bug.
+ // This helps us identify issues promptly.
+
+ // when use legacy planner, query an iceberg view will enter this
+ // we should set enable_fallback_to_original_planner=false
+ // so that it will throw exception by first planner
+ if (table.isView()) {
+ throw new UnsupportedOperationException("IcebergApiSource does not
support view");
+ }
this.icebergExtTable = table;
this.originTable =
Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
index f565f2795e8..c8e9c63d7b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
@@ -24,6 +24,9 @@ import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.iceberg.view.View;
+
+import java.util.Collections;
import java.util.List;
/**
@@ -141,4 +144,34 @@ public interface ExternalMetadataOps {
* close the connection, eg, to hms
*/
void close();
+
+ /**
+ * load an iceberg view.
+ * @param dbName
+ * @param viewName
+ * @return
+ */
+ default View loadView(String dbName, String viewName) {
+ throw new UnsupportedOperationException("Load view is not supported.");
+ }
+
+ /**
+ * Check if an Iceberg view exists.
+ * @param dbName
+ * @param viewName
+ * @return
+ */
+ default boolean viewExists(String dbName, String viewName) {
+ throw new UnsupportedOperationException("View is not supported.");
+ }
+
+ /**
+ * List all views under a specific database.
+ * @param db
+ * @return
+ */
+ default List<String> listViewNames(String db) {
+ return Collections.emptyList();
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 3c071058cf9..584cf5e1ebc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.analysis;
+import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.catalog.AggStateType;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
@@ -35,6 +36,7 @@ import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.nereids.CTEContext;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.SqlCacheContext;
@@ -405,7 +407,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
String hiveCatalog = hmsTable.getCatalog().getName();
String hiveDb = hmsTable.getDatabase().getFullName();
String ddlSql = hmsTable.getViewText();
- Plan hiveViewPlan = parseAndAnalyzeHiveView(
+ Plan hiveViewPlan = parseAndAnalyzeExternalView(
hmsTable, hiveCatalog, hiveDb, ddlSql,
cascadesContext);
return new LogicalSubQueryAlias<>(qualifiedTableName,
hiveViewPlan);
}
@@ -424,6 +426,33 @@ public class BindRelation extends OneAnalysisRuleFactory {
Optional.ofNullable(unboundRelation.getScanParams()));
}
case ICEBERG_EXTERNAL_TABLE:
+ IcebergExternalTable icebergExternalTable =
(IcebergExternalTable) table;
+ if (Config.enable_query_iceberg_views &&
icebergExternalTable.isView()) {
+ Optional<TableSnapshot> tableSnapshot =
unboundRelation.getTableSnapshot();
+ if (tableSnapshot.isPresent()) {
+ // iceberg view not supported with snapshot
time/version travel
+ // note that enable_fallback_to_original_planner
should be set with false
+ // or else this exception will not be thrown
+ // because legacy planner will retry and thrown
other exception
+ throw new UnsupportedOperationException(
+ "iceberg view not supported with snapshot
time/version travel");
+ }
+ isView = true;
+ String icebergCatalog =
icebergExternalTable.getCatalog().getName();
+ String icebergDb =
icebergExternalTable.getDatabase().getFullName();
+ String ddlSql = icebergExternalTable.getViewText();
+ Plan icebergViewPlan =
parseAndAnalyzeExternalView(icebergExternalTable,
+ icebergCatalog, icebergDb, ddlSql,
cascadesContext);
+ return new LogicalSubQueryAlias<>(qualifiedTableName,
icebergViewPlan);
+ }
+ if (icebergExternalTable.isView()) {
+ throw new UnsupportedOperationException(
+ "please set enable_query_iceberg_views=true to
enable query iceberg views");
+ }
+ return new
LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table,
+ qualifierWithoutTableName,
unboundRelation.getTableSample(),
+ unboundRelation.getTableSnapshot(),
+ Optional.ofNullable(unboundRelation.getScanParams()));
case PAIMON_EXTERNAL_TABLE:
case MAX_COMPUTE_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
@@ -462,15 +491,17 @@ public class BindRelation extends OneAnalysisRuleFactory {
}
}
- private Plan parseAndAnalyzeHiveView(
- HMSExternalTable table, String hiveCatalog, String hiveDb, String
ddlSql, CascadesContext cascadesContext) {
+ private Plan parseAndAnalyzeExternalView(
+ ExternalTable table, String externalCatalog, String externalDb,
+ String ddlSql, CascadesContext cascadesContext) {
ConnectContext ctx = cascadesContext.getConnectContext();
String previousCatalog = ctx.getCurrentCatalog().getName();
String previousDb = ctx.getDatabase();
String convertedSql = SqlDialectHelper.convertSqlByDialect(ddlSql,
ctx.getSessionVariable());
- // change catalog and db to hive catalog and db, so that we can parse
and analyze the view sql in hive context.
- ctx.changeDefaultCatalog(hiveCatalog);
- ctx.setDatabase(hiveDb);
+ // change catalog and db to external catalog and db,
+ // so that we can parse and analyze the view sql in external context.
+ ctx.changeDefaultCatalog(externalCatalog);
+ ctx.setDatabase(externalDb);
try {
return parseAndAnalyzeView(table, convertedSql, cascadesContext);
} finally {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java
index 5bc0c803cb9..29746fc34e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java
@@ -59,6 +59,9 @@ public class IcebergTableSink extends
BaseExternalTableDataSink {
public IcebergTableSink(IcebergExternalTable targetTable) {
super();
+ if (targetTable.isView()) {
+ throw new UnsupportedOperationException("Write data to iceberg
view is not supported");
+ }
this.targetTable = targetTable;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 945feebd3b4..196ce304ee9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -208,6 +208,7 @@ import
org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergUtils;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
import org.apache.doris.job.manager.JobManager;
import org.apache.doris.load.DeleteHandler;
@@ -1218,6 +1219,13 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
return;
}
+ if ((table.getType() == TableType.ICEBERG_EXTERNAL_TABLE)
+ && ((IcebergExternalTable) table).isView()) {
+ rows.add(Arrays.asList(table.getName(),
+ IcebergUtils.showCreateView(((IcebergExternalTable)
table))));
+ resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
+ return;
+ }
List<String> createTableStmt = Lists.newArrayList();
Env.getDdlStmt(null, null, table, createTableStmt, null, null,
false,
true /* hide password */, false, -1L,
showStmt.isNeedBriefDdl(), false);
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_view_query_p0.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_view_query_p0.out
new file mode 100644
index 00000000000..480a2a74156
Binary files /dev/null and
b/regression-test/data/external_table_p0/iceberg/test_iceberg_view_query_p0.out
differ
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_view_query_p0.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_view_query_p0.groovy
new file mode 100644
index 00000000000..a855ed38af1
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_view_query_p0.groovy
@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_view_query_p0",
"p0,external,iceberg,external_docker,external_docker_iceberg") {
+
+ String enableIcebergTest =
context.config.otherConfigs.get("enableIcebergTest")
+ // if (enableIcebergTest == null ||
!enableIcebergTest.equalsIgnoreCase("true")) {
+ // This is suit can not be run currently because only hive catalog can
support view.
+ // but we can only create view in rest catalog
+ // will be fixed later
+ if (true) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ try {
+ String hms_port = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String hdfs_port = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ //create iceberg hms catalog
+ String iceberg_catalog_name = "test_iceberg_view_query_p0"
+ sql """drop catalog if exists ${iceberg_catalog_name}"""
+ sql """create catalog if not exists ${iceberg_catalog_name} properties
(
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='hms',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'use_meta_cache' = 'true'
+ );"""
+
+ //using database and close planner fallback
+ sql """use `${iceberg_catalog_name}`.`test_db`"""
+ sql """set enable_fallback_to_original_planner=false;"""
+
+ // run all suites
+ def q01 = {
+ order_qt_q01 """select * from v_with_partitioned_table order by
col1"""
+ order_qt_q02 """select * from v_with_unpartitioned_table order by
col1"""
+ order_qt_q03 """select * from v_with_partitioned_column order by
col5"""
+ }
+
+ def q02 = {
+ order_qt_q01 """select count(*) from v_with_partitioned_table"""
+ order_qt_q02 """select count(*) from v_with_unpartitioned_table"""
+ order_qt_q03 """select count(*) from v_with_partitioned_column"""
+ }
+
+ def q03 = {
+ order_qt_q01 """select col1,col2,col3,col4 from
v_with_partitioned_table order by col1"""
+ order_qt_q02 """select col5 from v_with_partitioned_table order by
col5"""
+ }
+
+ def q04 = {
+ order_qt_q01 """describe v_with_partitioned_table"""
+ order_qt_q02 """describe v_with_unpartitioned_table"""
+ order_qt_q03 """describe v_with_partitioned_column"""
+ }
+
+ q01()
+ q02()
+ q03()
+ q04()
+
+ def result1 = sql """explain verbose select * from
v_with_partitioned_table"""
+ assertTrue(result1.contains('t_partitioned_table'))
+ def result2 = sql """explain verbose select * from
v_with_unpartitioned_table"""
+ assertTrue(result2.contains('v_with_unpartitioned_table'))
+ def result3 = sql """explain verbose select * from
v_with_partitioned_column"""
+ assertTrue(result3.contains('t_partitioned_table'))
+
+ def result4 = sql """explain verbose select count(*) from
v_with_partitioned_table"""
+ assertTrue(result4.contains('t_partitioned_table'))
+ def result5 = sql """explain verbose select count(*) from
v_with_unpartitioned_table"""
+ assertTrue(result5.contains('t_unpartitioned_table'))
+ def result6 = sql """explain verbose select count(*) from
v_with_partitioned_column"""
+ assertTrue(result6.contains('t_partitioned_table'))
+
+ def result7 = sql """explain verbose select col1,col2,col3,col4 from
v_with_partitioned_table"""
+ assertTrue(result7.contains('t_partitioned_table'))
+ def result8 = sql """explain verbose select col5 from
v_with_partitioned_table"""
+ assertTrue(result8.contains('t_partitioned_table'))
+
+ def result9 = sql """show create table v_with_partitioned_table"""
+ assertTrue(result9.contains('v_with_partitioned_table'))
+ def result10 = sql """show create table v_with_unpartitioned_table"""
+ assertTrue(result10.contains('v_with_unpartitioned_table'))
+ def result11 = sql """show create table v_with_partitioned_column"""
+ assertTrue(result11.contains('v_with_partitioned_column'))
+
+ def result12 = sql """show create view v_with_partitioned_table"""
+ assertTrue(result12.contains('v_with_partitioned_table'))
+ def result13 = sql """show create view v_with_unpartitioned_table"""
+ assertTrue(result13.contains('v_with_unpartitioned_table'))
+ def result14 = sql """show create view v_with_partitioned_column"""
+ assertTrue(result14.contains('v_with_partitioned_column'))
+
+ try {
+ sql """select * from v_with_partitioned_table FOR TIME AS OF
'2025-06-11 20:17:01' order by col1 limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select * from v_with_partitioned_table FOR VERSION AS OF
5497706844625725452 order by col1 limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_partitioned_table FOR TIME AS
OF '2025-06-11 20:17:01'"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_partitioned_table FOR VERSION
AS OF 5497706844625725452"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+
+ try {
+ sql """select * from v_with_unpartitioned_table FOR TIME AS OF
'2025-06-11 20:17:01' order by col1 limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select * from v_with_unpartitioned_table FOR VERSION AS OF
5497706844625725452 order by col1 limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_unpartitioned_table FOR TIME AS
OF '2025-06-11 20:17:01'"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_unpartitioned_table FOR VERSION
AS OF 5497706844625725452"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+
+ try {
+ sql """select * from v_with_partitioned_column FOR TIME AS OF
'2025-06-11 20:17:01' order by col5 limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select * from v_with_partitioned_column FOR VERSION AS OF
5497706844625725452 order by col5 limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_partitioned_column FOR TIME AS
OF '2025-06-11 20:17:01'"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_partitioned_column FOR VERSION
AS OF 5497706844625725452"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+
+ try {
+ sql """select * from v_with_joint_table FOR TIME AS OF '2025-06-11
20:17:01' order by sale_date limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select * from v_with_joint_table FOR VERSION AS OF
5497706844625725452 order by sale_date limit 10"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_joint_table FOR TIME AS OF
'2025-06-11 20:17:01'"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+ try {
+ sql """select count(*) from v_with_joint_table FOR VERSION AS OF
5497706844625725452"""
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("iceberg view not supported
with snapshot time/version travel"), e.getMessage())
+ }
+
+ sql """drop view v_with_partitioned_table"""
+ sql """drop view v_with_unpartitioned_table"""
+ sql """drop view v_with_partitioned_column"""
+
+ sql """drop catalog if exists ${iceberg_catalog_name}"""
+ } finally {
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
