This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 6302cdadd IMPALA-14665: Upgrade Iceberg to 1.10.1
6302cdadd is described below
commit 6302cdadde0a6627761f267bb060f7947a20334a
Author: Zoltan Borok-Nagy <[email protected]>
AuthorDate: Tue Nov 18 15:08:32 2025 +0100
IMPALA-14665: Upgrade Iceberg to 1.10.1
This patch upgrades Iceberg to 1.10.1. It also temporarily disables
CDP_ICEBERG_VERSION until it is also upgraded to 1.10.1.
Iceberg 1.10 depends on newer Avro and Parquet versions than what
Impala depends. To overcome this (without upgrading other Impala
dependencies and disable their CDP versions), we introduce a new
artifact: impala-iceberg-runtime. It shades the dependencies of the
Iceberg libary.
Test changes:
* Schema of a few metadata tables have changed, new fields added
* Before 1.10, a DELETE operation was stored as "overwrite" in Iceberg
snapshot metadata. Now it is correctly stored as "delete".
* API changes in IcebergRestCatalogTest
* Snappy compression for Parquet is always set by Impala if not
specified otherwise
Generated-by: Gemini Pro
Change-Id: I7e6f8e7e8e9e85faa992d7cc0165c92687d13bc0
Reviewed-on: http://gerrit.cloudera.org:8080/23945
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
bin/impala-config.sh | 5 +-
fe/pom.xml | 10 +-
.../apache/impala/analysis/CreateTableStmt.java | 3 +
.../iceberg/ImpalaIcebergDeleteOrphanFiles.java | 6 +-
.../java/org/apache/impala/service/Frontend.java | 3 +
.../org/apache/impala/planner/PlannerTest.java | 2 +-
java/datagenerator/pom.xml | 7 +
java/external-frontend/pom.xml | 6 +
java/iceberg-rest-catalog-test/pom.xml | 32 ++++-
.../iceberg/rest/IcebergRestCatalogTest.java | 16 +--
java/pom.xml | 12 +-
java/puffin-data-generator/pom.xml | 9 +-
java/shaded-deps/hive-exec/pom.xml | 23 ++--
java/shaded-deps/impala-iceberg-runtime/pom.xml | 150 +++++++++++++++++++++
java/shaded-deps/s3a-aws-sdk/pom.xml | 23 ++--
.../queries/QueryTest/iceberg-metadata-tables.test | 132 ++++++++++--------
.../queries/QueryTest/show-create-table.test | 3 +
tests/query_test/test_iceberg.py | 12 +-
18 files changed, 333 insertions(+), 121 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index d480ccc44..8fffa4ba0 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -264,7 +264,7 @@ export APACHE_MIRROR
export APACHE_AVRO_JAVA_VERSION=1.11.1
export APACHE_HADOOP_VERSION=3.4.1
export APACHE_HBASE_VERSION=2.6.0
-export APACHE_ICEBERG_VERSION=1.5.2
+export APACHE_ICEBERG_VERSION=1.10.1
export APACHE_KNOX_VERSION=2.0.0
export APACHE_ORC_JAVA_VERSION=1.8.3
export APACHE_PARQUET_VERSION=1.12.3
@@ -420,7 +420,8 @@ else
export IMPALA_HADOOP_URL=${CDP_HADOOP_URL-}
export IMPALA_HBASE_VERSION=${CDP_HBASE_VERSION}
export IMPALA_HBASE_URL=${CDP_HBASE_URL-}
- export IMPALA_ICEBERG_VERSION=${CDP_ICEBERG_VERSION}
+ # TODO(IMPALA-14727): Re-enable CDP_ICEBERG_VERSION once it is at least
1.10.1.
+ export IMPALA_ICEBERG_VERSION=${APACHE_ICEBERG_VERSION}
export IMPALA_ICEBERG_URL=${CDP_ICEBERG_URL-}
export IMPALA_KNOX_VERSION=${CDP_KNOX_VERSION}
export IMPALA_ORC_JAVA_VERSION=${CDP_ORC_JAVA_VERSION}
diff --git a/fe/pom.xml b/fe/pom.xml
index 7f9b9d312..61d376fea 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -343,13 +343,9 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-hive-runtime</artifactId>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-iceberg-runtime</artifactId>
+ <version>${project.version}</version>
</dependency>
<!-- Needed for reading Iceberg Puffin files. -->
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
index 6f19cd230..a67302df8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
@@ -662,6 +662,9 @@ public class CreateTableStmt extends StatementBase
implements SingleTableStmt {
putGeneratedProperty(IcebergTable.KEY_STORAGE_HANDLER,
IcebergTable.ICEBERG_STORAGE_HANDLER);
putGeneratedProperty(TableProperties.ENGINE_HIVE_ENABLED, "true");
+ if (!getTblProperties().containsKey("write.parquet.compression-codec")) {
+ putGeneratedProperty("write.parquet.compression-codec", "snappy");
+ }
addMergeOnReadPropertiesIfNeeded();
String fileformat =
getTblProperties().get(IcebergTable.ICEBERG_FILE_FORMAT);
diff --git
a/fe/src/main/java/org/apache/impala/catalog/iceberg/ImpalaIcebergDeleteOrphanFiles.java
b/fe/src/main/java/org/apache/impala/catalog/iceberg/ImpalaIcebergDeleteOrphanFiles.java
index 3b38fce2a..4d6ab3afb 100644
---
a/fe/src/main/java/org/apache/impala/catalog/iceberg/ImpalaIcebergDeleteOrphanFiles.java
+++
b/fe/src/main/java/org/apache/impala/catalog/iceberg/ImpalaIcebergDeleteOrphanFiles.java
@@ -37,9 +37,9 @@ import org.apache.iceberg.Table;
import org.apache.iceberg.TableScan;
import org.apache.iceberg.actions.DeleteOrphanFiles;
import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-import
org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.MoreExecutors;
import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java
b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 894a53c20..f9f8184f4 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -2944,6 +2944,9 @@ public class Frontend {
String positionalDeleteFiles =
Long.toString(metrics.positionalDeleteFiles().value());
addInfoString(profile, ScanMetrics.POSITIONAL_DELETE_FILES,
positionalDeleteFiles);
+
+ String deleteVectors = Long.toString(metrics.dvs().value());
+ addInfoString(profile, ScanMetrics.DVS, deleteVectors);
}
private TExecRequest doCreateExecRequest(CompilerFactory compilerFactory,
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index 2f6780608..50ba8f4b7 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -1713,7 +1713,7 @@ public class PlannerTest extends PlannerTestBase {
*/
@Test
public void testIcebergScanMetricsResultCardinality() {
- final int expectedMethodNumber = 16;
+ final int expectedMethodNumber = 17;
Method[] methods = ScanMetricsResult.class.getDeclaredMethods();
diff --git a/java/datagenerator/pom.xml b/java/datagenerator/pom.xml
index 8831c474f..78eb20047 100644
--- a/java/datagenerator/pom.xml
+++ b/java/datagenerator/pom.xml
@@ -58,6 +58,13 @@ under the License.
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.12.1</version>
+ </dependency>
+
</dependencies>
<build>
diff --git a/java/external-frontend/pom.xml b/java/external-frontend/pom.xml
index 61428891c..a13a2cf65 100644
--- a/java/external-frontend/pom.xml
+++ b/java/external-frontend/pom.xml
@@ -37,6 +37,12 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-frontend</artifactId>
<version>5.0.0-SNAPSHOT</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-iceberg-runtime</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/java/iceberg-rest-catalog-test/pom.xml
b/java/iceberg-rest-catalog-test/pom.xml
index 3e3b2c5d9..8da2b2b7d 100644
--- a/java/iceberg-rest-catalog-test/pom.xml
+++ b/java/iceberg-rest-catalog-test/pom.xml
@@ -44,6 +44,12 @@ under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -52,22 +58,36 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-api</artifactId>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-iceberg-runtime</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
+ <classifier>tests</classifier>
</dependency>
<dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-core</artifactId>
- <version>${iceberg.version}</version>
- <classifier>tests</classifier>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>11.0.26</version>
</dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <version>11.0.26</version>
+ </dependency>
+
+ <dependency>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+
</dependencies>
<build>
diff --git
a/java/iceberg-rest-catalog-test/src/main/java/org/apache/iceberg/rest/IcebergRestCatalogTest.java
b/java/iceberg-rest-catalog-test/src/main/java/org/apache/iceberg/rest/IcebergRestCatalogTest.java
index 3073adeb8..3e71c9930 100644
---
a/java/iceberg-rest-catalog-test/src/main/java/org/apache/iceberg/rest/IcebergRestCatalogTest.java
+++
b/java/iceberg-rest-catalog-test/src/main/java/org/apache/iceberg/rest/IcebergRestCatalogTest.java
@@ -92,17 +92,13 @@ public class IcebergRestCatalogTest {
RESTCatalogAdapter adapter = new RESTCatalogAdapter(catalog) {
@Override
public <T extends RESTResponse> T execute(
- RESTCatalogAdapter.HTTPMethod method,
- String path,
- Map<String, String> queryParams,
- Object body,
+ HTTPRequest request,
Class<T> responseType,
- Map<String, String> headers,
- Consumer<ErrorResponse> errorHandler) {
- Object request = roundTripSerialize(body, "request");
- T response =
- super.execute(
- method, path, queryParams, request, responseType, headers,
errorHandler);
+ Consumer<ErrorResponse> errorHandler,
+ Consumer<Map<String, String>> responseHeaders) {
+ Object body = roundTripSerialize(request.body(), "request");
+ HTTPRequest req =
ImmutableHTTPRequest.builder().from(request).body(body).build();
+ T response = super.execute(req, responseType, errorHandler,
responseHeaders);
return roundTripSerialize(response, "response");
}
};
diff --git a/java/pom.xml b/java/pom.xml
index 196b1e6d5..29c1de9e9 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -482,14 +482,9 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-api</artifactId>
- <version>${iceberg.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-hive-runtime</artifactId>
- <version>${iceberg.version}</version>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-iceberg-runtime</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
@@ -611,6 +606,7 @@ under the License.
<module>impala-package</module>
<module>query-event-hook-api</module>
<module>shaded-deps/hive-exec</module>
+ <module>shaded-deps/impala-iceberg-runtime</module>
<module>shaded-deps/s3a-aws-sdk</module>
<module>TableFlattener</module>
<module>test-hive-udfs</module>
diff --git a/java/puffin-data-generator/pom.xml
b/java/puffin-data-generator/pom.xml
index 9f892d756..74a8d8a20 100644
--- a/java/puffin-data-generator/pom.xml
+++ b/java/puffin-data-generator/pom.xml
@@ -45,12 +45,9 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-hive-runtime</artifactId>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-iceberg-runtime</artifactId>
+ <version>${project.version}</version>
</dependency>
<!-- Needed for reading Iceberg Puffin files. -->
diff --git a/java/shaded-deps/hive-exec/pom.xml
b/java/shaded-deps/hive-exec/pom.xml
index cf9e57abf..6ebdf5d18 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -1,16 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
diff --git a/java/shaded-deps/impala-iceberg-runtime/pom.xml
b/java/shaded-deps/impala-iceberg-runtime/pom.xml
new file mode 100644
index 000000000..a66d034fa
--- /dev/null
+++ b/java/shaded-deps/impala-iceberg-runtime/pom.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-parent</artifactId>
+ <version>5.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>impala-iceberg-runtime</artifactId>
+ <packaging>jar</packaging>
+
+
+ <properties>
+ <iceberg.parquet.version>1.16.0</iceberg.parquet.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-api</artifactId>
+ <version>${iceberg.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-core</artifactId>
+ <version>${iceberg.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-mr</artifactId>
+ <version>${iceberg.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-parquet</artifactId>
+ <version>${iceberg.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-hive-metastore</artifactId>
+ <version>${iceberg.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <version>${iceberg.parquet.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-column</artifactId>
+ <version>${iceberg.parquet.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <version>${iceberg.parquet.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.12.1</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
+ <version>5.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5</artifactId>
+ <version>5.4</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.5.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <includes>
+ <include>org.apache.iceberg:*</include>
+ <include>org.apache.avro:*</include>
+ <include>org.apache.parquet:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.avro</pattern>
+ <shadedPattern>impala.shaded.iceberg.avro</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.parquet</pattern>
+ <shadedPattern>impala.shaded.iceberg.parquet</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>shaded.parquet</pattern>
+
<shadedPattern>impala.shaded.iceberg.parquet.shaded</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/java/shaded-deps/s3a-aws-sdk/pom.xml
b/java/shaded-deps/s3a-aws-sdk/pom.xml
index 42803d669..5d4b5b359 100644
--- a/java/shaded-deps/s3a-aws-sdk/pom.xml
+++ b/java/shaded-deps/s3a-aws-sdk/pom.xml
@@ -1,16 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
index 9b6033668..bdf9fb041 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
@@ -21,28 +21,28 @@ INT,BIGINT,BIGINT,BIGINT,STRING,STRING
---- QUERY
select * from functional_parquet.iceberg_query_metadata.`files`;
---- RESULTS
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,NULL,'NULL',NULL,NULL,'{.*}'
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
---- QUERY
select * from functional_parquet.iceberg_query_metadata.data_files;
---- RESULTS
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
---- QUERY
select * from functional_parquet.iceberg_query_metadata.delete_files;
---- RESULTS
-row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,'{.*}'
+row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,NULL,'NULL',NULL,NULL,'{.*}'
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
---- QUERY
select * from functional_parquet.iceberg_query_metadata.history;
@@ -75,7 +75,7 @@ select * from
functional_parquet.iceberg_query_metadata.snapshots;
row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,NULL,'append','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,\d+,'append','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,\d+,'append','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
-row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,\d+,'overwrite','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
+row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,\d+,'delete','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
---- TYPES
TIMESTAMP,BIGINT,BIGINT,STRING,STRING,STRING
====
@@ -108,28 +108,28 @@ BIGINT, INT, BIGINT, BIGINT, INT, BIGINT, INT, TIMESTAMP,
BIGINT
---- QUERY
select * from functional_parquet.iceberg_query_metadata.all_data_files;
---- RESULTS
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
---- QUERY
select * from functional_parquet.iceberg_query_metadata.all_delete_files;
---- RESULTS
-row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,'{.*}'
+row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,NULL,'NULL',NULL,NULL,'{.*}'
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
---- QUERY
select * from functional_parquet.iceberg_query_metadata.all_files;
---- RESULTS
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,'{.*}'
-row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL,'{.*}'
+row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,NULL,'NULL',NULL,NULL,'{.*}'
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
---- QUERY
select * from functional_parquet.iceberg_query_metadata.all_manifests;
@@ -221,11 +221,11 @@ BIGINT
---- QUERY
# Test BIGINT
select * from functional_parquet.iceberg_query_metadata.history
-where snapshot_id = $OVERWRITE_SNAPSHOT_ID;
+where snapshot_id = $DELETE_SNAPSHOT_ID;
---- RESULTS
# Example:
# 2023-08-16 12:18:15.523000000,9046920472784493998,8491702501245661704,true
-row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,$OVERWRITE_SNAPSHOT_ID,\d+,true
+row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,$DELETE_SNAPSHOT_ID,\d+,true
---- TYPES
TIMESTAMP,BIGINT,BIGINT,BOOLEAN
====
@@ -246,38 +246,38 @@ TIMESTAMP,BIGINT,BIGINT,BOOLEAN
---- QUERY
# Test STRING
select * from functional_parquet.iceberg_query_metadata.snapshots
-where operation = 'overwrite';
+where operation = 'delete';
---- RESULTS
# Example:
# 2023-08-16
12:18:15.322000000,8491702501245661704,NULL,'append','hdfs://localhost:20500/test-warehouse/functional_parquet.db/iceberg_test_metadata/metadata/snap-8491702501245661704-1-88a39285-529f-41a4-bd69-6d2560fac64e.avro',NULL
-row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,\d+,'overwrite','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
+row_regex:\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})?,\d+,\d+,'delete','$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/metadata/.*.avro','{.*}'
---- TYPES
TIMESTAMP,BIGINT,BIGINT,STRING,STRING,STRING
====
---- QUERY
# Test TIMESTAMP
select * from functional_parquet.iceberg_query_metadata.history
-where made_current_at = cast("$OVERWRITE_SNAPSHOT_TS" as timestamp);
+where made_current_at = cast("$DELETE_SNAPSHOT_TS" as timestamp);
---- RESULTS
-row_regex:$OVERWRITE_SNAPSHOT_TS,$OVERWRITE_SNAPSHOT_ID,\d+,true
+row_regex:$DELETE_SNAPSHOT_TS,$DELETE_SNAPSHOT_ID,\d+,true
---- TYPES
TIMESTAMP,BIGINT,BIGINT,BOOLEAN
====
---- QUERY
# Test conjunct slot materialization
select snapshot_id from functional_parquet.iceberg_query_metadata.snapshots
-where operation = 'overwrite';
+where operation = 'delete';
---- RESULTS
-$OVERWRITE_SNAPSHOT_ID
+$DELETE_SNAPSHOT_ID
---- TYPES
BIGINT
====
---- QUERY
# Test an expression rewrite: OR -> IN ()
select * from functional_parquet.iceberg_query_metadata.history
-where snapshot_id = $OVERWRITE_SNAPSHOT_ID or snapshot_id = 1;
+where snapshot_id = $DELETE_SNAPSHOT_ID or snapshot_id = 1;
---- RESULTS
-row_regex:$OVERWRITE_SNAPSHOT_TS,$OVERWRITE_SNAPSHOT_ID,\d+,true
+row_regex:$DELETE_SNAPSHOT_TS,$DELETE_SNAPSHOT_ID,\d+,true
---- TYPES
TIMESTAMP,BIGINT,BIGINT,BOOLEAN
====
@@ -339,9 +339,9 @@ BIGINT
---- QUERY
select a.snapshot_id from functional_parquet.iceberg_query_metadata.history a
join functional_parquet.iceberg_query_metadata.snapshots b on a.snapshot_id =
b.snapshot_id
-where a.snapshot_id = $OVERWRITE_SNAPSHOT_ID;
+where a.snapshot_id = $DELETE_SNAPSHOT_ID;
---- RESULTS
-$OVERWRITE_SNAPSHOT_ID
+$DELETE_SNAPSHOT_ID
---- TYPES
BIGINT
@@ -380,7 +380,7 @@ TIMESTAMP,BIGINT,BIGINT,BOOLEAN
####
====
---- QUERY
-select * from functional_parquet.iceberg_query_metadata.snapshots FOR
SYSTEM_VERSION AS OF $OVERWRITE_SNAPSHOT_ID;
+select * from functional_parquet.iceberg_query_metadata.snapshots FOR
SYSTEM_VERSION AS OF $DELETE_SNAPSHOT_ID;
---- CATCH
AnalysisException: FOR SYSTEM_VERSION AS OF clause is only supported for
Iceberg tables. functional_parquet.iceberg_query_metadata.SNAPSHOTS is not an
Iceberg table.
====
@@ -749,10 +749,10 @@ join functional_parquet.iceberg_query_metadata.snapshots s
on h.snapshot_id = s.snapshot_id
order by made_current_at;
---- RESULTS
-'append',true,regex:'{"added-data-files":"1","added-records":"1","added-files-size":"[1-9][0-9]*","changed-partition-count":"1","total-records":"1","total-files-size":"[1-9][0-9]*","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"}'
-'append',true,regex:'{"added-data-files":"1","added-records":"1","added-files-size":"[1-9][0-9]*","changed-partition-count":"1","total-records":"2","total-files-size":"[1-9][0-9]*","total-data-files":"2","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"}'
-'append',true,regex:'{"added-data-files":"1","added-records":"1","added-files-size":"[1-9][0-9]*","changed-partition-count":"1","total-records":"3","total-files-size":"[1-9][0-9]*","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"}'
-'overwrite',true,regex:'{"added-position-delete-files":"1","added-delete-files":"1","added-files-size":"[1-9][0-9]*","added-position-deletes":"1","changed-partition-count":"1","total-records":"3","total-files-size":"[1-9][0-9]*","total-data-files":"3","total-delete-files":"1","total-position-deletes":"1","total-equality-deletes":"0"}'
+'append',true,regex:'{"added-data-files":"1","added-records":"1","added-files-size":"[1-9][0-9]*","changed-partition-count":"1","total-records":"1","total-files-size":"[1-9][0-9]*","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache
Iceberg.*"}'
+'append',true,regex:'{"added-data-files":"1","added-records":"1","added-files-size":"[1-9][0-9]*","changed-partition-count":"1","total-records":"2","total-files-size":"[1-9][0-9]*","total-data-files":"2","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache
Iceberg.*"}'
+'append',true,regex:'{"added-data-files":"1","added-records":"1","added-files-size":"[1-9][0-9]*","changed-partition-count":"1","total-records":"3","total-files-size":"[1-9][0-9]*","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache
Iceberg.*"}'
+'delete',true,regex:'{"added-position-delete-files":"1","added-delete-files":"1","added-files-size":"[1-9][0-9]*","added-position-deletes":"1","changed-partition-count":"1","total-records":"3","total-files-size":"[1-9][0-9]*","total-data-files":"3","total-delete-files":"1","total-position-deletes":"1","total-equality-deletes":"0","iceberg-version":"Apache
Iceberg.*"}'
---- TYPES
STRING,BOOLEAN,STRING
====
@@ -797,9 +797,9 @@ STRING,STRING
# Filter out position delete files because they contain filenames that vary by
dataload.
select data_file from functional_parquet.iceberg_query_metadata.entries where
data_file.content != 1;
---- RESULTS : VERIFY_IS_SUBSET
-row_regex:'{"content":0,"file_path":".*/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*_data.0.parq","file_format":"PARQUET","spec_id":0,"record_count":1,"file_size_in_bytes":[1-9][0-9]*,"column_sizes":{1:47},"value_counts":{1:1},"null_value_counts":{1:0},"nan_value_counts":null,"lower_bounds":{1:"AwAAAA=="},"upper_bounds":{1:"AwAAAA=="},"key_metadata":null,"split_offsets":null,"equality_ids":null,"sort_order_id":0}'
-row_regex:'{"content":0,"file_path":".*/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*_data.0.parq","file_format":"PARQUET","spec_id":0,"record_count":1,"file_size_in_bytes":[1-9][0-9]*,"column_sizes":{1:47},"value_counts":{1:1},"null_value_counts":{1:0},"nan_value_counts":null,"lower_bounds":{1:"AgAAAA=="},"upper_bounds":{1:"AgAAAA=="},"key_metadata":null,"split_offsets":null,"equality_ids":null,"sort_order_id":0}'
-row_regex:'{"content":0,"file_path":".*/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*_data.0.parq","file_format":"PARQUET","spec_id":0,"record_count":1,"file_size_in_bytes":[1-9][0-9]*,"column_sizes":{1:47},"value_counts":{1:1},"null_value_counts":{1:0},"nan_value_counts":null,"lower_bounds":{1:"AQAAAA=="},"upper_bounds":{1:"AQAAAA=="},"key_metadata":null,"split_offsets":null,"equality_ids":null,"sort_order_id":0}'
+row_regex:'{"content":0,"file_path":".*/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*_data.0.parq","file_format":"PARQUET","spec_id":0,"record_count":1,"file_size_in_bytes":[1-9][0-9]*,"column_sizes":{1:47},"value_counts":{1:1},"null_value_counts":{1:0},"nan_value_counts":null,"lower_bounds":{1:"AwAAAA=="},"upper_bounds":{1:"AwAAAA=="},"key_metadata":null,"split_offsets":null,"equality_ids":null,"sort_order_id":0,"first_row_id":null,"referenced_data_file":null,"cont
[...]
+row_regex:'{"content":0,"file_path":".*/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*_data.0.parq","file_format":"PARQUET","spec_id":0,"record_count":1,"file_size_in_bytes":[1-9][0-9]*,"column_sizes":{1:47},"value_counts":{1:1},"null_value_counts":{1:0},"nan_value_counts":null,"lower_bounds":{1:"AgAAAA=="},"upper_bounds":{1:"AgAAAA=="},"key_metadata":null,"split_offsets":null,"equality_ids":null,"sort_order_id":0,"first_row_id":null,"referenced_data_file":null,"cont
[...]
+row_regex:'{"content":0,"file_path":".*/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*_data.0.parq","file_format":"PARQUET","spec_id":0,"record_count":1,"file_size_in_bytes":[1-9][0-9]*,"column_sizes":{1:47},"value_counts":{1:1},"null_value_counts":{1:0},"nan_value_counts":null,"lower_bounds":{1:"AQAAAA=="},"upper_bounds":{1:"AQAAAA=="},"key_metadata":null,"split_offsets":null,"equality_ids":null,"sort_order_id":0,"first_row_id":null,"referenced_data_file":null,"cont
[...]
---- TYPES
STRING
====
@@ -807,12 +807,12 @@ STRING
# Filter out position delete files because they contain filenames that vary by
dataload.
select * from functional_parquet.iceberg_v2_delete_both_eq_and_pos.all_files
where content != 1;
---- RESULTS
-2,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet','PARQUET',0,2,697,'{1:40,3:66}','{1:2,3:2}','{1:0,3:0}','{}','{1:"AQAAAA==",3:"+EwAAA=="}','{1:"AgAAAA==",3:"+EwAAA=="}','NULL','[4]','[1,3]',0,'{"d":{"column_size":66,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-13"},"i":{"column_size":40,"value_count":2,"null_value_count":0,
[...]
-0,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet','PARQUET',0,2,885,'{1:40,2:62,3:40}','{1:2,2:2,3:2}','{1:0,2:0,3:0}','{}','{1:"AgAAAA==",2:"c3RyMl91cGRhdGVk",3:"+EwAAA=="}','{1:"AwAAAA==",2:"c3RyMw==",3:"Ak0AAA=="}','NULL','[4]','NULL',0,'{"d":{"column_size":40,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-23"},"i":{"column_
[...]
-0,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet','PARQUET',0,2,898,'{1:40,2:54,3:66}','{1:2,2:2,3:2}','{1:0,2:0,3:0}','{}','{1:"AQAAAA==",2:"c3RyMQ==",3:"+EwAAA=="}','{1:"AgAAAA==",2:"c3RyMg==",3:"+EwAAA=="}','NULL','[4]','NULL',0,'{"d":{"column_size":66,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-13"},"i":{"column_size":40
[...]
-2,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet','PARQUET',0,2,657,'{1:40,3:40}','{1:2,3:2}','{1:0,3:0}','{}','{1:"AgAAAA==",3:"+EwAAA=="}','{1:"AwAAAA==",3:"Ak0AAA=="}','NULL','[4]','[1,3]',0,'{"d":{"column_size":40,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-23"},"i":{"column_size":40,"value_count":2,"null_value_count":0,
[...]
+2,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet','PARQUET',0,2,697,'{1:40,3:66}','{1:2,3:2}','{1:0,3:0}','{}','{1:"AQAAAA==",3:"+EwAAA=="}','{1:"AgAAAA==",3:"+EwAAA=="}','NULL','[4]','[1,3]',0,NULL,'NULL',NULL,NULL,'{"d":{"column_size":66,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-13"},"i":{"column_size":40,"value_count":2
[...]
+0,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet','PARQUET',0,2,885,'{1:40,2:62,3:40}','{1:2,2:2,3:2}','{1:0,2:0,3:0}','{}','{1:"AgAAAA==",2:"c3RyMl91cGRhdGVk",3:"+EwAAA=="}','{1:"AwAAAA==",2:"c3RyMw==",3:"Ak0AAA=="}','NULL','[4]','NULL',0,NULL,'NULL',NULL,NULL,'{"d":{"column_size":40,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023
[...]
+0,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet','PARQUET',0,2,898,'{1:40,2:54,3:66}','{1:2,2:2,3:2}','{1:0,2:0,3:0}','{}','{1:"AQAAAA==",2:"c3RyMQ==",3:"+EwAAA=="}','{1:"AgAAAA==",2:"c3RyMg==",3:"+EwAAA=="}','NULL','[4]','NULL',0,NULL,'NULL',NULL,NULL,'{"d":{"column_size":66,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-13"}
[...]
+2,'/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet','PARQUET',0,2,657,'{1:40,3:40}','{1:2,3:2}','{1:0,3:0}','{}','{1:"AgAAAA==",3:"+EwAAA=="}','{1:"AwAAAA==",3:"Ak0AAA=="}','NULL','[4]','[1,3]',0,NULL,'NULL',NULL,NULL,'{"d":{"column_size":40,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-12-13","upper_bound":"2023-12-23"},"i":{"column_size":40,"value_count":2
[...]
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,STRING
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT,STRING
====
####
@@ -877,6 +877,10 @@ describe functional_parquet.iceberg_query_metadata.`files`;
'split_offsets','array<bigint>','Splittable offsets','true'
'equality_ids','array<int>','Equality comparison field IDs','true'
'sort_order_id','int','Sort order ID','true'
+'first_row_id','bigint','The first row ID assigned to the first row in the
data file','true'
+'referenced_data_file','string','Fully qualified location (URI with FS scheme)
of a data file that all deletes reference','true'
+'content_offset','bigint','The offset in the file where the content
starts','true'
+'content_size_in_bytes','bigint','The length of referenced content stored in
the file','true'
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -900,6 +904,10 @@ describe
functional_parquet.iceberg_query_metadata.data_files;
'split_offsets','array<bigint>','Splittable offsets','true'
'equality_ids','array<int>','Equality comparison field IDs','true'
'sort_order_id','int','Sort order ID','true'
+'first_row_id','bigint','The first row ID assigned to the first row in the
data file','true'
+'referenced_data_file','string','Fully qualified location (URI with FS scheme)
of a data file that all deletes reference','true'
+'content_offset','bigint','The offset in the file where the content
starts','true'
+'content_size_in_bytes','bigint','The length of referenced content stored in
the file','true'
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -923,6 +931,10 @@ describe
functional_parquet.iceberg_query_metadata.delete_files;
'split_offsets','array<bigint>','Splittable offsets','true'
'equality_ids','array<int>','Equality comparison field IDs','true'
'sort_order_id','int','Sort order ID','true'
+'first_row_id','bigint','The first row ID assigned to the first row in the
data file','true'
+'referenced_data_file','string','Fully qualified location (URI with FS scheme)
of a data file that all deletes reference','true'
+'content_offset','bigint','The offset in the file where the content
starts','true'
+'content_size_in_bytes','bigint','The length of referenced content stored in
the file','true'
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -1024,6 +1036,10 @@ describe
functional_parquet.iceberg_query_metadata.all_data_files;
'split_offsets','array<bigint>','Splittable offsets','true'
'equality_ids','array<int>','Equality comparison field IDs','true'
'sort_order_id','int','Sort order ID','true'
+'first_row_id','bigint','The first row ID assigned to the first row in the
data file','true'
+'referenced_data_file','string','Fully qualified location (URI with FS scheme)
of a data file that all deletes reference','true'
+'content_offset','bigint','The offset in the file where the content
starts','true'
+'content_size_in_bytes','bigint','The length of referenced content stored in
the file','true'
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -1047,6 +1063,10 @@ describe
functional_parquet.iceberg_query_metadata.all_delete_files;
'split_offsets','array<bigint>','Splittable offsets','true'
'equality_ids','array<int>','Equality comparison field IDs','true'
'sort_order_id','int','Sort order ID','true'
+'first_row_id','bigint','The first row ID assigned to the first row in the
data file','true'
+'referenced_data_file','string','Fully qualified location (URI with FS scheme)
of a data file that all deletes reference','true'
+'content_offset','bigint','The offset in the file where the content
starts','true'
+'content_size_in_bytes','bigint','The length of referenced content stored in
the file','true'
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -1070,6 +1090,10 @@ describe
functional_parquet.iceberg_query_metadata.all_files;
'split_offsets','array<bigint>','Splittable offsets','true'
'equality_ids','array<int>','Equality comparison field IDs','true'
'sort_order_id','int','Sort order ID','true'
+'first_row_id','bigint','The first row ID assigned to the first row in the
data file','true'
+'referenced_data_file','string','Fully qualified location (URI with FS scheme)
of a data file that all deletes reference','true'
+'content_offset','bigint','The offset in the file where the content
starts','true'
+'content_size_in_bytes','bigint','The length of referenced content stored in
the file','true'
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -1100,7 +1124,7 @@ describe
functional_parquet.iceberg_query_metadata.all_entries;
'snapshot_id','bigint','','true'
'sequence_number','bigint','','true'
'file_sequence_number','bigint','','true'
-'data_file','struct<\n content:int comment ''contents of the file: 0=data,
1=position deletes, 2=equality deletes'',\n file_path:string comment
''location uri with fs scheme'',\n file_format:string comment ''file format
name: avro, orc, or parquet'',\n spec_id:int comment ''partition spec id'',\n
record_count:bigint comment ''number of records in the file'',\n
file_size_in_bytes:bigint comment ''total file size in bytes'',\n
column_sizes:map<int,bigint> comment ''map of column id [...]
+'data_file','struct<\n content:int comment ''contents of the file: 0=data,
1=position deletes, 2=equality deletes'',\n file_path:string comment
''location uri with fs scheme'',\n file_format:string comment ''file format
name: avro, orc, or parquet'',\n spec_id:int comment ''partition spec id'',\n
record_count:bigint comment ''number of records in the file'',\n
file_size_in_bytes:bigint comment ''total file size in bytes'',\n
column_sizes:map<int,bigint> comment ''map of column id [...]
'readable_metrics','struct<\n i:struct<\n column_size:bigint comment
''total size on disk'',\n value_count:bigint comment ''total count,
including null and nan'',\n null_value_count:bigint comment ''null value
count'',\n nan_value_count:bigint comment ''nan value count'',\n
lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper
bound''\n > comment ''metrics for column i''\n>','Column metrics in readable
form','true'
---- TYPES
STRING,STRING,STRING,STRING
@@ -1141,12 +1165,12 @@ AnalysisException: The SHOW METADATA TABLES statement
is only valid for Iceberg
# Expand a struct column using 'path.*' syntax.
select data_file.* from functional_parquet.iceberg_query_metadata.`entries`;
---- RESULTS
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0
-row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0
-row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL
+row_regex:0,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',0,NULL,'NULL',NULL,NULL
+row_regex:1,'$NAMENODE/test-warehouse/iceberg_test/metadata/iceberg_query_metadata/data/.*.parq','PARQUET',0,1,\d+,'{.*}','{.*}','{.*}','NULL','{.*}','{.*}','NULL','NULL','NULL',NULL,NULL,'NULL',NULL,NULL
---- TYPES
-INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT
+INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BINARY,STRING,STRING,INT,BIGINT,STRING,BIGINT,BIGINT
====
---- QUERY
# Join a metadata table with a random other table that has complex columns and
check that
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
index fd1b72433..f5e48af33 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
@@ -775,6 +775,7 @@ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.format.default'='orc',
+ 'write.parquet.compression-codec'='snappy',
'engine.hive.enabled'='true',
'iceberg.catalog'='hadoop.tables',
'write.delete.mode'='merge-on-read',
@@ -958,6 +959,7 @@ LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'write.format.default'='parquet',
+ 'write.parquet.compression-codec'='snappy',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'iceberg.catalog'='hadoop.tables',
@@ -1001,6 +1003,7 @@ LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'write.format.default'='parquet',
+ 'write.parquet.compression-codec'='snappy',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'iceberg.catalog'='ice_hadoop_cat',
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 87e8bddc4..e76c6c38b 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -1707,16 +1707,16 @@ class TestIcebergV2Table(IcebergTestSuite):
# Revisit this if 'batch_size' dimension size increase.
vector.unset_exec_option('batch_size')
with self.create_impala_client() as impalad_client:
- overwrite_snapshot_id = impalad_client.execute("select snapshot_id from "
+ delete_snapshot_id = impalad_client.execute("select snapshot_id from "
"functional_parquet.iceberg_query_metadata.snapshots "
- "where operation = 'overwrite';")
- overwrite_snapshot_ts = impalad_client.execute("select committed_at from
"
+ "where operation = 'delete';")
+ delete_snapshot_ts = impalad_client.execute("select committed_at from "
"functional_parquet.iceberg_query_metadata.snapshots "
- "where operation = 'overwrite';")
+ "where operation = 'delete';")
self.run_test_case('QueryTest/iceberg-metadata-tables', vector,
unique_database,
- test_file_vars={'$OVERWRITE_SNAPSHOT_ID':
str(overwrite_snapshot_id.data[0]),
- '$OVERWRITE_SNAPSHOT_TS':
str(overwrite_snapshot_ts.data[0])})
+ test_file_vars={'$DELETE_SNAPSHOT_ID':
str(delete_snapshot_id.data[0]),
+ '$DELETE_SNAPSHOT_TS':
str(delete_snapshot_ts.data[0])})
@SkipIf.not_hdfs
def test_missing_data_files(self, vector, unique_database):
