This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5ad73a6771e HIVE-29460: Iceberg: Support CLUSTERED BY for iceberg
tables (#6358)
5ad73a6771e is described below
commit 5ad73a6771e6d3a6f6d2d026b3e6345ff562419b
Author: kokila-19 <[email protected]>
AuthorDate: Fri May 8 17:58:07 2026 +0530
HIVE-29460: Iceberg: Support CLUSTERED BY for iceberg tables (#6358)
---
.../apache/iceberg/hive/HiveOperationsBase.java | 15 +
.../apache/iceberg/hive/HiveTableOperations.java | 16 +-
.../mr/hive/TestHiveIcebergClusteredBy.java | 185 ++++++
.../test/queries/positive/iceberg_clustered_by.q | 85 +++
.../positive/llap/iceberg_clustered_by.q.out | 685 +++++++++++++++++++++
.../test/resources/testconfiguration.properties | 2 +
.../desc/formatter/TextDescTableFormatter.java | 6 +-
7 files changed, 991 insertions(+), 3 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
index 937939b4816..3afff626bac 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
@@ -20,7 +20,9 @@
package org.apache.iceberg.hive;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@@ -161,6 +163,12 @@ default void persistTable(Table hmsTable, boolean
updateHiveTable, String metada
static StorageDescriptor storageDescriptor(
Schema schema, String location, boolean hiveEngineEnabled) {
+ return storageDescriptor(schema, location, hiveEngineEnabled, null, -1);
+ }
+
+ static StorageDescriptor storageDescriptor(
+ Schema schema, String location, boolean hiveEngineEnabled,
+ List<String> bucketCols, int numBuckets) {
final StorageDescriptor storageDescriptor = new StorageDescriptor();
storageDescriptor.setCols(HiveSchemaUtil.convert(schema));
storageDescriptor.setLocation(location);
@@ -178,6 +186,13 @@ static StorageDescriptor storageDescriptor(
}
storageDescriptor.setSerdeInfo(serDeInfo);
+
+ // Preserve Hive bucketing information if provided (for CLUSTERED BY
support)
+ if (CollectionUtils.isNotEmpty(bucketCols) && numBuckets > 0) {
+ storageDescriptor.setBucketCols(bucketCols);
+ storageDescriptor.setNumBuckets(numBuckets);
+ }
+
return storageDescriptor;
}
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index 0aeece27dba..104dd658408 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -20,6 +20,7 @@
package org.apache.iceberg.hive;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -30,6 +31,7 @@
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.iceberg.BaseMetastoreOperations;
import org.apache.iceberg.BaseMetastoreTableOperations;
@@ -178,11 +180,23 @@ protected void doCommit(TableMetadata base, TableMetadata
metadata) {
LOG.debug("Committing new table: {}", fullName);
}
+ // Preserve Hive bucketing (CLUSTERED BY) if present in the existing
table
+ StorageDescriptor sd = tbl.getSd();
+ List<String> bucketCols = null;
+ int numBuckets = -1;
+
+ if (sd != null) {
+ bucketCols = sd.getBucketCols();
+ numBuckets = sd.getNumBuckets();
+ }
+
tbl.setSd(
HiveOperationsBase.storageDescriptor(
metadata.schema(),
metadata.location(),
- hiveEngineEnabled)); // set to pickup any schema changes
+ hiveEngineEnabled, // set to pickup any schema changes
+ bucketCols,
+ numBuckets));
String metadataLocation =
tbl.getParameters().get(METADATA_LOCATION_PROP);
String baseMetadataLocation = base != null ? base.metadataFileLocation()
: null;
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergClusteredBy.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergClusteredBy.java
new file mode 100644
index 00000000000..3bc3c712722
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergClusteredBy.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.mr.hive.test.TestTables.TestTableType;
+import
org.apache.iceberg.mr.hive.test.utils.HiveIcebergStorageHandlerTestUtils;
+import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Verifies that Iceberg tables honor the {@code CLUSTERED BY} clause by
checking HMS metadata
+ * persistence, data readability and physical file distribution across buckets.
+ */
+public class TestHiveIcebergClusteredBy extends
HiveIcebergStorageHandlerWithEngineBase {
+
+ private static final String TABLE_NAME = "ice_clus_data_files";
+ private static final TableIdentifier TABLE_ID =
TableIdentifier.of("default", TABLE_NAME);
+ private static final int NUM_BUCKETS = 3;
+
+ private static final Map<Integer, Long> EXPECTED_BUCKET_RECORD_COUNTS =
buildExpectedBuckets();
+
+ private static Map<Integer, Long> buildExpectedBuckets() {
+ Map<Integer, Long> bucketCounts = new LinkedHashMap<>();
+ bucketCounts.put(0, 4L); // customer_id=2
+ bucketCounts.put(1, 3L); // customer_id=3
+ bucketCounts.put(2, 2L); // customer_id=1
+ return Collections.unmodifiableMap(bucketCounts);
+ }
+
+ @Parameters(name = "fileFormat={0}, catalog={1}, isVectorized={2},
formatVersion={3}")
+ public static Collection<Object[]> parameters() {
+ return HiveIcebergStorageHandlerWithEngineBase.getParameters(p ->
+ p.fileFormat() == FileFormat.PARQUET &&
+ p.testTableType() == TestTableType.HIVE_CATALOG &&
+ p.isVectorized());
+ }
+
+ private void setUpIcebergTable() {
+ shell.executeStatement(
+ "CREATE TABLE " + TABLE_NAME +
+ " (customer_id BIGINT, first_name STRING, last_name STRING) " +
+ "CLUSTERED BY (customer_id) INTO " + NUM_BUCKETS + " BUCKETS " +
+ "STORED BY ICEBERG STORED AS PARQUET " +
+ "TBLPROPERTIES ('format-version'='" + formatVersion + "')");
+
+ shell.executeStatement(testTables.getInsertQuery(
+ HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2, TABLE_ID,
false));
+ }
+
+ @Test
+ public void testHmsHasBucketMetadata() throws TException,
InterruptedException {
+ setUpIcebergTable();
+ validateHmsBucketMetadata(TABLE_NAME);
+ }
+
+ @Test
+ public void testTotalRowCountAfterClusteredInsert() {
+ setUpIcebergTable();
+ List<Object[]> result = shell.executeStatement("SELECT COUNT(*) FROM " +
TABLE_NAME);
+ long count = ((Number) result.getFirst()[0]).longValue();
+
Assert.assertEquals(HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2.size(),
count);
+ }
+
+ @Test
+ public void testIcebergDataFilesAfterClusteredInsert() throws IOException {
+ setUpIcebergTable();
+ org.apache.iceberg.Table icebergTable = testTables.loadTable(TABLE_ID);
+ Map<Integer, Long> actualBucketRecordCounts =
extractBucketRecordCounts(icebergTable);
+ Assert.assertEquals(EXPECTED_BUCKET_RECORD_COUNTS,
actualBucketRecordCounts);
+ }
+
+ /**
+ * Tests that CLUSTERED BY metadata and bucketing behavior are preserved
when migrating
+ * a native Hive table to Iceberg using ALTER TABLE CONVERT TO ICEBERG.
+ */
+ @Test
+ public void testNativeToIcebergMigrationWithClusteredBy() throws TException,
InterruptedException, IOException {
+ String migrationTableName = "migration_clustered_table";
+ TableIdentifier migrationTableId = TableIdentifier.of("default",
migrationTableName);
+
+ // Create a Native Hive table with CLUSTERED BY
+ shell.executeStatement(
+ "CREATE EXTERNAL TABLE " + migrationTableName +
+ " (customer_id BIGINT, first_name STRING, last_name STRING) " +
+ "CLUSTERED BY (customer_id) INTO " + NUM_BUCKETS + " BUCKETS " +
+ "STORED AS ORC");
+
+ // Insert initial records into the Native Hive table
+ List<org.apache.iceberg.data.Record> initialRecords =
+
HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2.subList(0, 4);
+ shell.executeStatement(testTables.getInsertQuery(initialRecords,
migrationTableId, false));
+
+ validateHmsBucketMetadata(migrationTableName);
+
+ List<Object[]> initialCount = shell.executeStatement("SELECT COUNT(*) FROM
" + migrationTableName);
+ Assert.assertEquals(4L, ((Number) initialCount.getFirst()[0]).longValue());
+
+ // Convert the Native Hive table to Iceberg table
+ shell.executeStatement("ALTER TABLE " + migrationTableName + " CONVERT TO
ICEBERG");
+
+ validateHmsBucketMetadata(migrationTableName);
+ Table hmsTable = shell.metastore().getTable("default", migrationTableName);
+ Assert.assertEquals("org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ hmsTable.getParameters().get("storage_handler"));
+
+ // Insert remaining records into the Iceberg table
+ List<org.apache.iceberg.data.Record> remainingRecords =
+
HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2.subList(4, 9);
+ shell.executeStatement(testTables.getInsertQuery(remainingRecords,
migrationTableId, false));
+
+ // Validate Iceberg table metadata and bucketing behavior
+ List<Object[]> finalCount = shell.executeStatement("SELECT COUNT(*) FROM "
+ migrationTableName);
+
Assert.assertEquals(HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2.size(),
+ ((Number) finalCount.getFirst()[0]).longValue());
+
+ org.apache.iceberg.Table icebergTable =
testTables.loadTable(migrationTableId);
+ Map<Integer, Long> bucketCounts = extractBucketRecordCounts(icebergTable);
+ Assert.assertEquals(EXPECTED_BUCKET_RECORD_COUNTS, bucketCounts);
+
+ shell.executeStatement("DROP TABLE IF EXISTS " + migrationTableName);
+ }
+
+ private void validateHmsBucketMetadata(String tableName) throws TException,
InterruptedException {
+ Table hmsTable = shell.metastore().getTable("default", tableName);
+ Assert.assertEquals(NUM_BUCKETS, hmsTable.getSd().getNumBuckets());
+ Assert.assertEquals(Collections.singletonList("customer_id"),
hmsTable.getSd().getBucketCols());
+ }
+
+ /**
+ * Extracts bucket ID from file names and counts records per bucket.
+ * Handles both Iceberg file names ({bucketId}-{attemptId}-...) and native
Hive file names ({bucketId}_0).
+ */
+ private Map<Integer, Long>
extractBucketRecordCounts(org.apache.iceberg.Table icebergTable) throws
IOException {
+ Map<Integer, Long> bucketRecordCounts = new LinkedHashMap<>();
+ try (CloseableIterable<FileScanTask> tasks =
icebergTable.newScan().planFiles()) {
+ for (FileScanTask task : tasks) {
+ String path = task.file().location();
+ String filename = path.substring(path.lastIndexOf('/') + 1);
+
+ int bucketId;
+ if (!filename.contains("-")) {
+ // Native Hive: 000000_0 → bucket 0
+ bucketId = Integer.parseInt(filename.split("_")[0]);
+ } else {
+ // Iceberg: 00000-0-... → bucket 0
+ bucketId = Integer.parseInt(filename.split("-")[0]);
+ }
+
+ // Sum records across multiple files in the same bucket (e.g.,
multiple inserts)
+ bucketRecordCounts.merge(bucketId, task.file().recordCount(),
Long::sum);
+ }
+ }
+ return bucketRecordCounts;
+ }
+}
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_clustered_by.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_clustered_by.q
new file mode 100644
index 00000000000..524ac931422
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_clustered_by.q
@@ -0,0 +1,85 @@
+-- Test Iceberg tables with CLUSTERED BY clause
+-- Verify that Iceberg tables follow Hive bucketing behavior for query
optimization
+
+-- SORT_QUERY_RESULTS
+-- Mask random uuid
+--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask a random snapshot id
+--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/
+-- Mask added file size
+--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask total file size
+--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask current-snapshot-timestamp-ms
+--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask removed file size
+--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask number of files
+--! qt:replace:/(\s+numFiles\s+)\S+(\s+)/$1#Masked#$2/
+-- Mask total data files
+--! qt:replace:/(\S\"total-data-files\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask iceberg version
+--!
qt:replace:/(\S\"iceberg-version\\\":\\\")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))(\\\")/$1#Masked#$3/
+
+-- Simple CREATE with CLUSTERED BY
+CREATE TABLE ice_bucketed_simple (
+ id int,
+ name string,
+ age int
+)
+CLUSTERED BY (id) INTO 4 BUCKETS
+STORED BY ICEBERG;
+
+DESCRIBE FORMATTED ice_bucketed_simple;
+
+EXPLAIN INSERT INTO ice_bucketed_simple VALUES (1, 'Alice', 25), (2, 'Bob',
30);
+
+INSERT INTO ice_bucketed_simple VALUES
+ (1, 'Alice', 25),
+ (2, 'Bob', 30),
+ (3, 'Carrie', 22),
+ (4, 'Danny', 28),
+ (5, 'Eve', 35),
+ (6, 'Frank', 40),
+ (7, 'Grace', 27),
+ (8, 'Henry', 33),
+ (9, 'Ivy', 29),
+ (10, 'Jack', 31);
+
+SELECT * FROM ice_bucketed_simple ORDER BY id;
+
+SELECT COUNT(*) from default.ice_bucketed_simple.files;
+
+-- CLUSTERED BY on multiple columns
+CREATE TABLE ice_bucketed_multi (
+ customer_id int,
+ order_id bigint,
+ product string
+)
+CLUSTERED BY (customer_id, order_id) INTO 8 BUCKETS
+STORED BY ICEBERG;
+
+DESCRIBE FORMATTED ice_bucketed_multi;
+
+EXPLAIN INSERT INTO ice_bucketed_multi VALUES (100, 1001, 'Widget'), (101,
1002, 'Gadget');
+
+-- Convert EXTERNAL Hive table with CLUSTERED BY to Iceberg
+CREATE EXTERNAL TABLE hive_bucketed (
+ product_id int,
+ category string,
+ price decimal(10,2)
+)
+CLUSTERED BY (product_id) INTO 3 BUCKETS
+STORED AS ORC;
+
+INSERT INTO hive_bucketed VALUES (1, 'Electronics', 299.99), (2, 'Books',
15.50);
+
+ALTER TABLE hive_bucketed CONVERT TO ICEBERG;
+
+DESCRIBE FORMATTED hive_bucketed;
+
+EXPLAIN INSERT INTO hive_bucketed VALUES (3, 'Furniture', 450.00);
+
+DROP TABLE IF EXISTS ice_bucketed_simple;
+DROP TABLE IF EXISTS ice_bucketed_multi;
+DROP TABLE IF EXISTS hive_bucketed;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out
new file mode 100644
index 00000000000..53b11da616c
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out
@@ -0,0 +1,685 @@
+PREHOOK: query: CREATE TABLE ice_bucketed_simple (
+ id int,
+ name string,
+ age int
+)
+CLUSTERED BY (id) INTO 4 BUCKETS
+STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_bucketed_simple
+POSTHOOK: query: CREATE TABLE ice_bucketed_simple (
+ id int,
+ name string,
+ age int
+)
+CLUSTERED BY (id) INTO 4 BUCKETS
+STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_bucketed_simple
+PREHOOK: query: DESCRIBE FORMATTED ice_bucketed_simple
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_bucketed_simple
+POSTHOOK: query: DESCRIBE FORMATTED ice_bucketed_simple
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_bucketed_simple
+# col_name data_type comment
+id int
+name string
+age int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"id\":\"true\",\"name\":\"true\"}}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"age\",\"required\":false,\"type\":\"int\"}]}
+ format-version 2
+ iceberg.orc.files.only false
+#### A masked pattern was here ####
+ numFiles #Masked#
+ numRows 0
+ parquet.compression zstd
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 0
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.merge.mode merge-on-read
+ write.metadata.delete-after-commit.enabled true
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Num Buckets: 4
+Bucket Columns: [id]
+Sort Columns: []
+PREHOOK: query: EXPLAIN INSERT INTO ice_bucketed_simple VALUES (1, 'Alice',
25), (2, 'Bob', 30)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_bucketed_simple
+POSTHOOK: query: EXPLAIN INSERT INTO ice_bucketed_simple VALUES (1, 'Alice',
25), (2, 'Bob', 30)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_bucketed_simple
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(1,'Alice',25),const
struct(2,'Bob',30)) (type: array<struct<col1:int,col2:string,col3:int>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 56 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 56 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string),
col3 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col2
(type: int)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0
(type: string), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.ice_bucketed_simple
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string), _col2
(type: int)
+ outputColumnNames: id, name, age
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(id), max(id), count(1), count(id),
compute_bit_vector_hll(id), max(length(name)), avg(COALESCE(length(name),0)),
count(name), compute_bit_vector_hll(name), min(age), max(age), count(age),
compute_bit_vector_hll(age)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 560 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 560 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int),
_col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type:
int), _col6 (type: struct<count:bigint,sum:double,input:int>), _col7 (type:
bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11
(type: bigint), _col12 (type: binary)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col0) (type:
bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary),
'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint),
COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary),
'LONG' (type: string), UDFToLong(_col9) (typ [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17
+ Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 794 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.ice_bucketed_simple
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: id, name, age
+ Column Types: int, string, int
+ Table: default.ice_bucketed_simple
+
+PREHOOK: query: INSERT INTO ice_bucketed_simple VALUES
+ (1, 'Alice', 25),
+ (2, 'Bob', 30),
+ (3, 'Carrie', 22),
+ (4, 'Danny', 28),
+ (5, 'Eve', 35),
+ (6, 'Frank', 40),
+ (7, 'Grace', 27),
+ (8, 'Henry', 33),
+ (9, 'Ivy', 29),
+ (10, 'Jack', 31)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_bucketed_simple
+POSTHOOK: query: INSERT INTO ice_bucketed_simple VALUES
+ (1, 'Alice', 25),
+ (2, 'Bob', 30),
+ (3, 'Carrie', 22),
+ (4, 'Danny', 28),
+ (5, 'Eve', 35),
+ (6, 'Frank', 40),
+ (7, 'Grace', 27),
+ (8, 'Henry', 33),
+ (9, 'Ivy', 29),
+ (10, 'Jack', 31)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_bucketed_simple
+PREHOOK: query: SELECT * FROM ice_bucketed_simple ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_bucketed_simple
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM ice_bucketed_simple ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_bucketed_simple
+#### A masked pattern was here ####
+1 Alice 25
+10 Jack 31
+2 Bob 30
+3 Carrie 22
+4 Danny 28
+5 Eve 35
+6 Frank 40
+7 Grace 27
+8 Henry 33
+9 Ivy 29
+PREHOOK: query: SELECT COUNT(*) from default.ice_bucketed_simple.files
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_bucketed_simple
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) from default.ice_bucketed_simple.files
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_bucketed_simple
+#### A masked pattern was here ####
+4
+PREHOOK: query: CREATE TABLE ice_bucketed_multi (
+ customer_id int,
+ order_id bigint,
+ product string
+)
+CLUSTERED BY (customer_id, order_id) INTO 8 BUCKETS
+STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_bucketed_multi
+POSTHOOK: query: CREATE TABLE ice_bucketed_multi (
+ customer_id int,
+ order_id bigint,
+ product string
+)
+CLUSTERED BY (customer_id, order_id) INTO 8 BUCKETS
+STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_bucketed_multi
+PREHOOK: query: DESCRIBE FORMATTED ice_bucketed_multi
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_bucketed_multi
+POSTHOOK: query: DESCRIBE FORMATTED ice_bucketed_multi
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_bucketed_multi
+# col_name data_type comment
+customer_id int
+order_id bigint
+product string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"customer_id\":\"true\",\"order_id\":\"true\",\"product\":\"true\"}}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"customer_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"order_id\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"product\",\"required\":false,\"type\":\"string\"}]}
+ format-version 2
+ iceberg.orc.files.only false
+#### A masked pattern was here ####
+ numFiles #Masked#
+ numRows 0
+ parquet.compression zstd
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 0
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.merge.mode merge-on-read
+ write.metadata.delete-after-commit.enabled true
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Num Buckets: 8
+Bucket Columns: [customer_id, order_id]
+Sort Columns: []
+PREHOOK: query: EXPLAIN INSERT INTO ice_bucketed_multi VALUES (100, 1001,
'Widget'), (101, 1002, 'Gadget')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_bucketed_multi
+POSTHOOK: query: EXPLAIN INSERT INTO ice_bucketed_multi VALUES (100, 1001,
'Widget'), (101, 1002, 'Gadget')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_bucketed_multi
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(100,1001,'Widget'),const
struct(101,1002,'Gadget')) (type: array<struct<col1:int,col2:int,col3:string>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 56 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 56 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), UDFToLong(col2) (type:
bigint), col3 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type:
bigint)
+ null sort order: aa
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int),
_col1 (type: bigint)
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int),
KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.ice_bucketed_multi
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: bigint), _col2
(type: string)
+ outputColumnNames: customer_id, order_id, product
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(customer_id), max(customer_id),
count(1), count(customer_id), compute_bit_vector_hll(customer_id),
min(order_id), max(order_id), count(order_id),
compute_bit_vector_hll(order_id), max(length(product)),
avg(COALESCE(length(product),0)), count(product),
compute_bit_vector_hll(product)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 568 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 568 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int),
_col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type:
bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary),
_col9 (type: int), _col10 (type: struct<count:bigint,sum:double,input:int>),
_col11 (type: bigint), _col12 (type: binary)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
min(VALUE._col5), max(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col0) (type:
bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary),
'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 -
_col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type:
bigint), _col8 (type: binary), 'STRING' (type: string),
UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_c [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17
+ Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 794 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.ice_bucketed_multi
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: customer_id, order_id, product
+ Column Types: int, bigint, string
+ Table: default.ice_bucketed_multi
+
+PREHOOK: query: CREATE EXTERNAL TABLE hive_bucketed (
+ product_id int,
+ category string,
+ price decimal(10,2)
+)
+CLUSTERED BY (product_id) INTO 3 BUCKETS
+STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive_bucketed
+POSTHOOK: query: CREATE EXTERNAL TABLE hive_bucketed (
+ product_id int,
+ category string,
+ price decimal(10,2)
+)
+CLUSTERED BY (product_id) INTO 3 BUCKETS
+STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive_bucketed
+PREHOOK: query: INSERT INTO hive_bucketed VALUES (1, 'Electronics', 299.99),
(2, 'Books', 15.50)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive_bucketed
+POSTHOOK: query: INSERT INTO hive_bucketed VALUES (1, 'Electronics', 299.99),
(2, 'Books', 15.50)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive_bucketed
+POSTHOOK: Lineage: hive_bucketed.category SCRIPT []
+POSTHOOK: Lineage: hive_bucketed.price SCRIPT []
+POSTHOOK: Lineage: hive_bucketed.product_id SCRIPT []
+PREHOOK: query: ALTER TABLE hive_bucketed CONVERT TO ICEBERG
+PREHOOK: type: ALTERTABLE_CONVERT
+PREHOOK: Input: default@hive_bucketed
+POSTHOOK: query: ALTER TABLE hive_bucketed CONVERT TO ICEBERG
+POSTHOOK: type: ALTERTABLE_CONVERT
+POSTHOOK: Input: default@hive_bucketed
+POSTHOOK: Output: default@hive_bucketed
+PREHOOK: query: DESCRIBE FORMATTED hive_bucketed
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hive_bucketed
+POSTHOOK: query: DESCRIBE FORMATTED hive_bucketed
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hive_bucketed
+# col_name data_type comment
+product_id int
+category string
+price decimal(10,2)
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"category\":\"true\",\"price\":\"true\",\"product_id\":\"true\"}}
+ EXTERNAL TRUE
+ MIGRATED_TO_ICEBERG true
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"product_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"category\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"price\",\"required\":false,\"type\":\"decimal(10,
2)\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"added-data-files\":\"2\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"#Masked#\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-timestamp-ms #Masked#
+ format-version 2
+ iceberg.orc.files.only true
+#### A masked pattern was here ####
+ numFiles #Masked#
+ numRows 2
+ parquet.compression zstd
+#### A masked pattern was here ####
+ rawDataSize 416
+ schema.name-mapping.default [ {
+ \"field-id\" : 1,
+ \"names\" : [ \"product_id\" ]
+ }, {
+ \"field-id\" : 2,
+ \"names\" : [ \"category\" ]
+ }, {
+ \"field-id\" : 3,
+ \"names\" : [ \"price\" ]
+ } ]
+ snapshot-count 1
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.format.default orc
+ write.merge.mode merge-on-read
+ write.metadata.delete-after-commit.enabled true
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Num Buckets: 3
+Bucket Columns: [product_id]
+Sort Columns: []
+PREHOOK: query: EXPLAIN INSERT INTO hive_bucketed VALUES (3, 'Furniture',
450.00)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive_bucketed
+POSTHOOK: query: EXPLAIN INSERT INTO hive_bucketed VALUES (3, 'Furniture',
450.00)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive_bucketed
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(3,'Furniture',450)) (type:
array<struct<col1:int,col2:string,col3:decimal(3,0)>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string),
CAST( col3 AS decimal(10,2)) (type: decimal(10,2))
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 112 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 112 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col2
(type: decimal(10,2))
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0
(type: string), VALUE._col1 (type: decimal(10,2))
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.hive_bucketed
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string), _col2
(type: decimal(10,2))
+ outputColumnNames: product_id, category, price
+ Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(product_id), max(product_id), count(1),
count(product_id), compute_bit_vector_hll(product_id), max(length(category)),
avg(COALESCE(length(category),0)), count(category),
compute_bit_vector_hll(category), min(price), max(price), count(price),
compute_bit_vector_hll(price)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 776 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 776 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int),
_col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type:
int), _col6 (type: struct<count:bigint,sum:double,input:int>), _col7 (type:
bigint), _col8 (type: binary), _col9 (type: decimal(10,2)), _col10 (type:
decimal(10,2)), _col11 (type: bigint), _col12 (type: binary)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 708 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col0) (type:
bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary),
'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint),
COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary),
'DECIMAL' (type: string), _col9 (type: decim [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17
+ Statistics: Num rows: 1 Data size: 1005 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 1005 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.hive_bucketed
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: product_id, category, price
+ Column Types: int, string, decimal(10,2)
+ Table: default.hive_bucketed
+
+PREHOOK: query: DROP TABLE IF EXISTS ice_bucketed_simple
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_bucketed_simple
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_bucketed_simple
+POSTHOOK: query: DROP TABLE IF EXISTS ice_bucketed_simple
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ice_bucketed_simple
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_bucketed_simple
+PREHOOK: query: DROP TABLE IF EXISTS ice_bucketed_multi
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_bucketed_multi
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_bucketed_multi
+POSTHOOK: query: DROP TABLE IF EXISTS ice_bucketed_multi
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ice_bucketed_multi
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_bucketed_multi
+PREHOOK: query: DROP TABLE IF EXISTS hive_bucketed
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive_bucketed
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive_bucketed
+POSTHOOK: query: DROP TABLE IF EXISTS hive_bucketed
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive_bucketed
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive_bucketed
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 64a761acfdd..214afa40d6a 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -429,6 +429,7 @@ iceberg.llap.query.files=\
iceberg_bucket_map_join_7.q,\
iceberg_bucket_map_join_8.q,\
iceberg_clustered.q,\
+ iceberg_clustered_by.q,\
iceberg_create_locally_ordered_table.q,\
iceberg_create_locally_zordered_table.q,\
iceberg_merge_delete_files.q,\
@@ -480,6 +481,7 @@ iceberg.llap.only.query.files=\
iceberg_bucket_map_join_7.q,\
iceberg_bucket_map_join_8.q,\
iceberg_clustered.q,\
+ iceberg_clustered_by.q,\
iceberg_create_locally_ordered_table.q,\
iceberg_create_locally_zordered_table.q,\
iceberg_merge_delete_files.q,\
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
index b1dd9738572..75f39291cd0 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
@@ -311,9 +311,11 @@ private void getStorageDescriptorInfo(StringBuilder
tableInfo, Table table, Stor
formatOutput("InputFormat:", storageDesc.getInputFormat(), tableInfo);
formatOutput("OutputFormat:", storageDesc.getOutputFormat(), tableInfo);
formatOutput("Compressed:", storageDesc.isCompressed() ? "Yes" : "No",
tableInfo);
+ // Show bucket columns for Native table or Iceberg table with CLUSTERED BY
(Hive-style bucketing)
+ boolean hasClusteredBy = storageDesc.getNumBuckets() > 0
+ && CollectionUtils.isNotEmpty(storageDesc.getBucketCols());
if (!table.isNonNative() || table.getStorageHandler() == null ||
- !table.getStorageHandler().supportsPartitionTransform()) {
- // The Iceberg partition transform already contains the bucketing
information, and these are not relevant there
+ !table.getStorageHandler().supportsPartitionTransform() ||
hasClusteredBy) {
formatOutput("Num Buckets:",
String.valueOf(storageDesc.getNumBuckets()), tableInfo);
formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(),
tableInfo);
}
