This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new fe67670b195 HIVE-28077: Iceberg: Major QB Compaction on partition
level (Dmitriy Fingerman, reviewed by Denys Kuzmenko)
fe67670b195 is described below
commit fe67670b195f1037431a0588dfbbd4c2cd84e277
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Thu Apr 25 04:40:54 2024 -0400
HIVE-28077: Iceberg: Major QB Compaction on partition level (Dmitriy
Fingerman, reviewed by Denys Kuzmenko)
Closes #5123
---
.../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 41 ++
.../apache/iceberg/mr/hive/IcebergTableUtil.java | 2 +-
.../compaction/IcebergMajorQueryCompactor.java | 63 ++-
.../iceberg_major_compaction_single_partition.q | 69 ++++
...iceberg_major_compaction_single_partition.q.out | 451 +++++++++++++++++++++
.../test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/ql/DriverUtils.java | 3 +
.../ql/ddl/table/partition/PartitionUtils.java | 2 +-
.../compact/AlterTableCompactOperation.java | 7 +
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 27 +-
.../hive/ql/metadata/HiveStorageHandler.java | 36 ++
.../service/CompactionExecutorFactory.java | 2 +-
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 5 +
14 files changed, 692 insertions(+), 19 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 5503e4e01c9..1d8ebdcd0bc 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -480,6 +480,8 @@ public enum ErrorMsg {
INVALID_METADATA_TABLE_NAME(10430, "Invalid metadata table name {0}.", true),
TABLE_META_REF_NOT_SUPPORTED(10431, "Table Meta Ref extension is not
supported for table {0}.", true),
COMPACTION_REFUSED(10432, "Compaction request for {0}.{1}{2} is refused,
details: {3}.", true),
+ COMPACTION_PARTITION_EVOLUTION(10438, "Compaction for {0}.{1} on partition
level is not allowed on a table that has undergone partition evolution", true),
+ COMPACTION_NON_IDENTITY_PARTITION_SPEC(10439, "Compaction for {0}.{1} is not
supported on the table with non-identity partition spec", true),
CBO_IS_REQUIRED(10433,
"The following functionality requires CBO (" +
HiveConf.ConfVars.HIVE_CBO_ENABLED.varname + "): {0}", true),
CTLF_UNSUPPORTED_FORMAT(10434, "CREATE TABLE LIKE FILE is not supported by
the ''{0}'' file format", true),
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index cf59e18685a..fbc8ec2e01a 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -57,12 +57,14 @@ import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
@@ -1920,6 +1922,45 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
.anyMatch(id -> id < table.spec().specId());
}
+ private boolean
isIdentityPartitionTable(org.apache.hadoop.hive.ql.metadata.Table table) {
+ return
getPartitionTransformSpec(table).stream().map(TransformSpec::getTransformType)
+ .allMatch(type -> type == TransformSpec.TransformType.IDENTITY);
+ }
+
+ @Override
+ public Optional<ErrorMsg> isEligibleForCompaction(
+ org.apache.hadoop.hive.ql.metadata.Table table, Map<String, String>
partitionSpec) {
+ if (partitionSpec != null) {
+ Table icebergTable = IcebergTableUtil.getTable(conf, table.getTTable());
+ if (hasUndergonePartitionEvolution(icebergTable)) {
+ return Optional.of(ErrorMsg.COMPACTION_PARTITION_EVOLUTION);
+ }
+ if (!isIdentityPartitionTable(table)) {
+ return Optional.of(ErrorMsg.COMPACTION_NON_IDENTITY_PARTITION_SPEC);
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public List<Partition>
getPartitions(org.apache.hadoop.hive.ql.metadata.Table table,
+ Map<String, String> partitionSpec) throws SemanticException {
+ return getPartitionNames(table, partitionSpec).stream()
+ .map(partName -> new DummyPartition(table, partName,
partitionSpec)).collect(Collectors.toList());
+ }
+
+ @Override
+ public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table,
+ Map<String, String> partitionSpec) throws SemanticException {
+ validatePartSpec(table, partitionSpec);
+ try {
+ String partName = Warehouse.makePartName(partitionSpec, false);
+ return new DummyPartition(table, partName, partitionSpec);
+ } catch (MetaException e) {
+ throw new SemanticException("Unable to construct name for dummy
partition due to: ", e);
+ }
+ }
+
/**
* Returns a list of partitions which are corresponding to the table based
on the partition spec provided.
* @param hmsTable A Hive table instance.
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
index a47f6fbfdef..17576edc9f2 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
@@ -85,7 +85,7 @@ public class IcebergTableUtil {
return getTable(configuration, properties, skipCache);
}
- static Table getTable(Configuration configuration,
org.apache.hadoop.hive.metastore.api.Table hmsTable) {
+ public static Table getTable(Configuration configuration,
org.apache.hadoop.hive.metastore.api.Table hmsTable) {
return getTable(configuration, hmsTable, false);
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
index e3dba519dc9..d55172715f7 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
@@ -20,15 +20,26 @@
package org.apache.iceberg.mr.hive.compaction;
import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Context.RewritePolicy;
import org.apache.hadoop.hive.ql.DriverUtils;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext;
import org.apache.hadoop.hive.ql.txn.compactor.QueryCompactor;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hive.iceberg.org.apache.orc.storage.common.TableName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,22 +55,52 @@ public class IcebergMajorQueryCompactor extends
QueryCompactor {
Map<String, String> tblProperties = context.getTable().getParameters();
LOG.debug("Initiating compaction for the {} table", compactTableName);
- String compactionQuery = String.format("insert overwrite table %s select *
from %<s",
- compactTableName);
+ HiveConf conf = new HiveConf(context.getConf());
+ String partSpec = context.getCompactionInfo().partName;
+ String compactionQuery;
+
+ if (partSpec == null) {
+ HiveConf.setVar(conf, ConfVars.REWRITE_POLICY,
RewritePolicy.ALL_PARTITIONS.name());
+ compactionQuery = String.format("insert overwrite table %s select * from
%<s", compactTableName);
+ } else {
+ org.apache.hadoop.hive.ql.metadata.Table table =
Hive.get(conf).getTable(context.getTable().getDbName(),
+ context.getTable().getTableName());
+ Map<String, String> partSpecMap = new LinkedHashMap<>();
+ Warehouse.makeSpecFromName(partSpecMap, new Path(partSpec), null);
+
+ List<FieldSchema> partitionKeys =
table.getStorageHandler().getPartitionKeys(table);
+ List<String> partValues = partitionKeys.stream().map(
+ fs -> String.join("=", HiveUtils.unparseIdentifier(fs.getName()),
+
TypeInfoUtils.convertStringToLiteralForSQL(partSpecMap.get(fs.getName()),
+ ((PrimitiveTypeInfo)
TypeInfoUtils.getTypeInfoFromTypeString(fs.getType())).getPrimitiveCategory())
+ )
+ ).collect(Collectors.toList());
+
+ String queryFields = table.getCols().stream()
+ .map(FieldSchema::getName)
+ .filter(col -> !partSpecMap.containsKey(col))
+ .collect(Collectors.joining(","));
+
+ compactionQuery = String.format("insert overwrite table %1$s
partition(%2$s) select %4$s from %1$s where %3$s",
+ compactTableName,
+ StringUtils.join(partValues, ","),
+ StringUtils.join(partValues, " and "),
+ queryFields);
+ }
+
+ SessionState sessionState = setupQueryCompactionSession(conf,
context.getCompactionInfo(), tblProperties);
+ String compactionTarget = "table " +
HiveUtils.unparseIdentifier(compactTableName) +
+ (partSpec != null ? ", partition " +
HiveUtils.unparseIdentifier(partSpec) : "");
- SessionState sessionState = setupQueryCompactionSession(context.getConf(),
- context.getCompactionInfo(), tblProperties);
- HiveConf.setVar(context.getConf(), ConfVars.REWRITE_POLICY,
RewritePolicy.ALL_PARTITIONS.name());
try {
- DriverUtils.runOnDriver(context.getConf(), sessionState,
compactionQuery);
- LOG.info("Completed compaction for table {}", compactTableName);
+ DriverUtils.runOnDriver(conf, sessionState, compactionQuery);
+ LOG.info("Completed compaction for {}", compactionTarget);
+ return true;
} catch (HiveException e) {
- LOG.error("Error doing query based {} compaction",
RewritePolicy.ALL_PARTITIONS.name(), e);
- throw new RuntimeException(e);
+ LOG.error("Failed compacting {}", compactionTarget, e);
+ throw e;
} finally {
sessionState.setCompaction(false);
}
-
- return true;
}
}
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
new file mode 100644
index 00000000000..35ae60459b5
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
@@ -0,0 +1,69 @@
+-- SORT_QUERY_RESULTS
+-- Mask neededVirtualColumns due to non-strict order
+--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+-- Mask random uuid
+--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask a random snapshot id
+--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/
+-- Mask added file size
+--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask total file size
+--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask removed file size
+--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask current-snapshot-timestamp-ms
+--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
+--!
qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
+-- Mask compaction id as they will be allocated in parallel threads
+--! qt:replace:/^[0-9]/#Masked#/
+
+set hive.llap.io.enabled=true;
+set hive.vectorized.execution.enabled=true;
+set hive.optimize.shared.work.merge.ts.schema=true;
+
+create table ice_orc_wo_evo (
+ first_name string,
+ last_name string
+ )
+partitioned by (dept_id bigint,
+ city string,
+ registration_date date)
+stored by iceberg stored as orc
+tblproperties ('format-version'='2');
+
+insert into ice_orc_wo_evo VALUES ('fn1','ln1',1,'London','2024-03-11');
+insert into ice_orc_wo_evo VALUES ('fn2','ln2',1,'London','2024-03-11');
+insert into ice_orc_wo_evo VALUES ('fn3','ln3',1,'London','2024-03-11');
+insert into ice_orc_wo_evo VALUES ('fn4','ln4',1,'London','2024-03-11');
+insert into ice_orc_wo_evo VALUES ('fn5','ln5',2,'Paris','2024-02-16');
+insert into ice_orc_wo_evo VALUES ('fn6','ln6',2,'Paris','2024-02-16');
+insert into ice_orc_wo_evo VALUES ('fn7','ln7',2,'Paris','2024-02-16');
+
+update ice_orc_wo_evo set last_name = 'ln1a' where first_name='fn1';
+update ice_orc_wo_evo set last_name = 'ln2a' where first_name='fn2';
+update ice_orc_wo_evo set last_name = 'ln3a' where first_name='fn3';
+update ice_orc_wo_evo set last_name = 'ln4a' where first_name='fn4';
+update ice_orc_wo_evo set last_name = 'ln5a' where first_name='fn5';
+update ice_orc_wo_evo set last_name = 'ln6a' where first_name='fn6';
+update ice_orc_wo_evo set last_name = 'ln7a' where first_name='fn7';
+
+delete from ice_orc_wo_evo where last_name in ('ln1a', 'ln2a', 'ln7a');
+
+select * from ice_orc_wo_evo;
+describe formatted ice_orc_wo_evo;
+
+explain alter table ice_orc_wo_evo PARTITION (dept_id=1, city='London',
registration_date='2024-03-11') COMPACT 'major' and wait;
+alter table ice_orc_wo_evo PARTITION (dept_id=1, city='London',
registration_date='2024-03-11') COMPACT 'major' and wait;
+
+select * from ice_orc_wo_evo;
+describe formatted ice_orc_wo_evo;
+show compactions;
+
+explain alter table ice_orc_wo_evo PARTITION (dept_id=2, city='Paris',
registration_date='2024-02-16') COMPACT 'major' and wait;
+alter table ice_orc_wo_evo PARTITION (dept_id=2, city='Paris',
registration_date='2024-02-16') COMPACT 'major' and wait;
+
+select * from ice_orc_wo_evo;
+describe formatted ice_orc_wo_evo;
+show compactions;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
new file mode 100644
index 00000000000..b2aeeac2e50
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
@@ -0,0 +1,451 @@
+PREHOOK: query: create table ice_orc_wo_evo (
+ first_name string,
+ last_name string
+ )
+partitioned by (dept_id bigint,
+ city string,
+ registration_date date)
+stored by iceberg stored as orc
+tblproperties ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: create table ice_orc_wo_evo (
+ first_name string,
+ last_name string
+ )
+partitioned by (dept_id bigint,
+ city string,
+ registration_date date)
+stored by iceberg stored as orc
+tblproperties ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn1','ln1',1,'London','2024-03-11')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn1','ln1',1,'London','2024-03-11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn2','ln2',1,'London','2024-03-11')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn2','ln2',1,'London','2024-03-11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn3','ln3',1,'London','2024-03-11')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn3','ln3',1,'London','2024-03-11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn4','ln4',1,'London','2024-03-11')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn4','ln4',1,'London','2024-03-11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn5','ln5',2,'Paris','2024-02-16')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn5','ln5',2,'Paris','2024-02-16')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn6','ln6',2,'Paris','2024-02-16')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn6','ln6',2,'Paris','2024-02-16')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: insert into ice_orc_wo_evo VALUES
('fn7','ln7',2,'Paris','2024-02-16')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: insert into ice_orc_wo_evo VALUES
('fn7','ln7',2,'Paris','2024-02-16')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln1a' where
first_name='fn1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln1a' where
first_name='fn1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln2a' where
first_name='fn2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln2a' where
first_name='fn2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln3a' where
first_name='fn3'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln3a' where
first_name='fn3'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln4a' where
first_name='fn4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln4a' where
first_name='fn4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln5a' where
first_name='fn5'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln5a' where
first_name='fn5'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln6a' where
first_name='fn6'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln6a' where
first_name='fn6'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: update ice_orc_wo_evo set last_name = 'ln7a' where
first_name='fn7'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: query: update ice_orc_wo_evo set last_name = 'ln7a' where
first_name='fn7'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+POSTHOOK: Output: default@ice_orc_wo_evo
+PREHOOK: query: delete from ice_orc_wo_evo where last_name in ('ln1a', 'ln2a',
'ln7a')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+POSTHOOK: query: delete from ice_orc_wo_evo where last_name in ('ln1a',
'ln2a', 'ln7a')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+PREHOOK: query: select * from ice_orc_wo_evo
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc_wo_evo
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+fn3 ln3a 1 London 2024-03-11
+fn4 ln4a 1 London 2024-03-11
+fn5 ln5a 2 Paris 2024-02-16
+fn6 ln6a 2 Paris 2024-02-16
+PREHOOK: query: describe formatted ice_orc_wo_evo
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: query: describe formatted ice_orc_wo_evo
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_orc_wo_evo
+# col_name data_type comment
+first_name string
+last_name string
+dept_id bigint
+city string
+registration_date date
+
+# Partition Transform Information
+# col_name transform_type
+dept_id IDENTITY
+city IDENTITY
+registration_date IDENTITY
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"},{\"id\":5,\"name\":\"registration_date\",\"required\":false,\"type\":\"date\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"deleted-data-files\":\"3\",\"deleted-records\":\"3\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"7\",\"total-position-deletes\":\"7\",\"total-equality-deletes\":\"0\"}
+ current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1002}]}
+ format-version 2
+ iceberg.orc.files.only true
+#### A masked pattern was here ####
+ numFiles 11
+ numRows 11
+ parquet.compression zstd
+#### A masked pattern was here ####
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 15
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.format.default orc
+ write.merge.mode merge-on-read
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: explain alter table ice_orc_wo_evo PARTITION (dept_id=1,
city='London', registration_date='2024-03-11') COMPACT 'major' and wait
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output:
default@ice_orc_wo_evo@dept_id=1/city=London/registration_date=2024-03-11
+POSTHOOK: query: explain alter table ice_orc_wo_evo PARTITION (dept_id=1,
city='London', registration_date='2024-03-11') COMPACT 'major' and wait
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output:
default@ice_orc_wo_evo@dept_id=1/city=London/registration_date=2024-03-11
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: major
+ table name: default.ice_orc_wo_evo
+ numberOfBuckets: 0
+ partition spec:
+ city London
+ dept_id 1
+ registration_date 2024-03-11
+ table name: default.ice_orc_wo_evo
+ blocking: true
+
+PREHOOK: query: alter table ice_orc_wo_evo PARTITION (dept_id=1,
city='London', registration_date='2024-03-11') COMPACT 'major' and wait
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output:
default@ice_orc_wo_evo@dept_id=1/city=London/registration_date=2024-03-11
+POSTHOOK: query: alter table ice_orc_wo_evo PARTITION (dept_id=1,
city='London', registration_date='2024-03-11') COMPACT 'major' and wait
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output:
default@ice_orc_wo_evo@dept_id=1/city=London/registration_date=2024-03-11
+PREHOOK: query: select * from ice_orc_wo_evo
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc_wo_evo
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+fn3 ln3a 1 London 2024-03-11
+fn4 ln4a 1 London 2024-03-11
+fn5 ln5a 2 Paris 2024-02-16
+fn6 ln6a 2 Paris 2024-02-16
+PREHOOK: query: describe formatted ice_orc_wo_evo
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: query: describe formatted ice_orc_wo_evo
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_orc_wo_evo
+# col_name data_type comment
+first_name string
+last_name string
+dept_id bigint
+city string
+registration_date date
+
+# Partition Transform Information
+# col_name transform_type
+dept_id IDENTITY
+city IDENTITY
+registration_date IDENTITY
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"city\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"registration_date\":\"true\"}}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"},{\"id\":5,\"name\":\"registration_date\",\"required\":false,\"type\":\"date\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"replace-partitions\":\"true\",\"added-data-files\":\"1\",\"deleted-data-files\":\"6\",\"removed-position-delete-files\":\"4\",\"removed-delete-files\":\"4\",\"added-records\":\"2\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"4\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"6\",\"total-delete-files\":\"3\",\"t
[...]
+ current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1002}]}
+ format-version 2
+ iceberg.orc.files.only true
+#### A masked pattern was here ####
+ numFiles 6
+ numRows 4
+ parquet.compression zstd
+#### A masked pattern was here ####
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 16
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.format.default orc
+ write.merge.mode merge-on-read
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: show compactions
+PREHOOK: type: SHOW COMPACTIONS
+POSTHOOK: query: show compactions
+POSTHOOK: type: SHOW COMPACTIONS
+CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
+#Masked# default ice_orc_wo_evo
dept_id=1/city=London/registration_date=2024-03-11 MAJOR succeeded
#Masked# manual default 0 0 0 ---
+PREHOOK: query: explain alter table ice_orc_wo_evo PARTITION (dept_id=2,
city='Paris', registration_date='2024-02-16') COMPACT 'major' and wait
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output:
default@ice_orc_wo_evo@dept_id=2/city=Paris/registration_date=2024-02-16
+POSTHOOK: query: explain alter table ice_orc_wo_evo PARTITION (dept_id=2,
city='Paris', registration_date='2024-02-16') COMPACT 'major' and wait
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output:
default@ice_orc_wo_evo@dept_id=2/city=Paris/registration_date=2024-02-16
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: major
+ table name: default.ice_orc_wo_evo
+ numberOfBuckets: 0
+ partition spec:
+ city Paris
+ dept_id 2
+ registration_date 2024-02-16
+ table name: default.ice_orc_wo_evo
+ blocking: true
+
+PREHOOK: query: alter table ice_orc_wo_evo PARTITION (dept_id=2, city='Paris',
registration_date='2024-02-16') COMPACT 'major' and wait
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc_wo_evo
+PREHOOK: Output:
default@ice_orc_wo_evo@dept_id=2/city=Paris/registration_date=2024-02-16
+POSTHOOK: query: alter table ice_orc_wo_evo PARTITION (dept_id=2,
city='Paris', registration_date='2024-02-16') COMPACT 'major' and wait
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: Output:
default@ice_orc_wo_evo@dept_id=2/city=Paris/registration_date=2024-02-16
+PREHOOK: query: select * from ice_orc_wo_evo
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc_wo_evo
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_wo_evo
+#### A masked pattern was here ####
+fn3 ln3a 1 London 2024-03-11
+fn4 ln4a 1 London 2024-03-11
+fn5 ln5a 2 Paris 2024-02-16
+fn6 ln6a 2 Paris 2024-02-16
+PREHOOK: query: describe formatted ice_orc_wo_evo
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_orc_wo_evo
+POSTHOOK: query: describe formatted ice_orc_wo_evo
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_orc_wo_evo
+# col_name data_type comment
+first_name string
+last_name string
+dept_id bigint
+city string
+registration_date date
+
+# Partition Transform Information
+# col_name transform_type
+dept_id IDENTITY
+city IDENTITY
+registration_date IDENTITY
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"city\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"registration_date\":\"true\"}}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"},{\"id\":5,\"name\":\"registration_date\",\"required\":false,\"type\":\"date\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"replace-partitions\":\"true\",\"added-data-files\":\"1\",\"deleted-data-files\":\"5\",\"removed-position-delete-files\":\"3\",\"removed-delete-files\":\"3\",\"added-records\":\"2\",\"deleted-records\":\"5\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"3\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"t
[...]
+ current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1002}]}
+ format-version 2
+ iceberg.orc.files.only true
+#### A masked pattern was here ####
+ numFiles 2
+ numRows 4
+ parquet.compression zstd
+#### A masked pattern was here ####
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 17
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.format.default orc
+ write.merge.mode merge-on-read
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: show compactions
+PREHOOK: type: SHOW COMPACTIONS
+POSTHOOK: query: show compactions
+POSTHOOK: type: SHOW COMPACTIONS
+CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
+#Masked# default ice_orc_wo_evo
dept_id=1/city=London/registration_date=2024-03-11 MAJOR succeeded
#Masked# manual default 0 0 0 ---
+#Masked# default ice_orc_wo_evo
dept_id=2/city=Paris/registration_date=2024-02-16 MAJOR succeeded
#Masked# manual default 0 0 0 ---
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 02627bba691..50e16a912f3 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -423,6 +423,7 @@ iceberg.llap.query.compactor.files=\
iceberg_major_compaction_partition_evolution.q,\
iceberg_major_compaction_partitioned.q,\
iceberg_major_compaction_schema_evolution.q,\
+ iceberg_major_compaction_single_partition.q,\
iceberg_major_compaction_unpartitioned.q,\
iceberg_optimize_table_unpartitioned.q
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
index 948dff7c389..e56b099f055 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
@@ -128,6 +128,9 @@ public final class DriverUtils {
}
SessionState.setCurrentSessionState(sessionState);
}
+ else {
+ sessionState.setConf(conf);
+ }
return sessionState;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
index 28cd5019c1a..d5128fe79f4 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
@@ -95,7 +95,7 @@ public final class PartitionUtils {
throws SemanticException {
Partition partition;
try {
- partition = db.getPartition(table, partitionSpec, false);
+ partition = db.getPartition(table, partitionSpec);
} catch (Exception e) {
throw new SemanticException(toMessage(ErrorMsg.INVALID_PARTITION,
partitionSpec), e);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
index 19f4b48d8c6..c2daa1200d9 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
@@ -61,6 +61,13 @@ public class AlterTableCompactOperation extends
DDLOperation<AlterTableCompactDe
if (!AcidUtils.isTransactionalTable(table) &&
!AcidUtils.isNonNativeAcidTable(table)) {
throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED,
table.getDbName(), table.getTableName());
}
+
+ if (table.getStorageHandler() != null) {
+ Optional<ErrorMsg> error =
table.getStorageHandler().isEligibleForCompaction(table,
desc.getPartitionSpec());
+ if (error.isPresent()) {
+ throw new HiveException(error.get(), table.getDbName(),
table.getTableName());
+ }
+ }
Map<String, org.apache.hadoop.hive.metastore.api.Partition> partitionMap =
convertPartitionsFromThriftToDB(getPartitions(table, desc, context));
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 3ee27912e9e..781ed16219f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3660,6 +3660,15 @@ private void constructOneLBLocationMap(FileStatus fSta,
throw new HiveException(e);
}
}
+
+ public Partition getPartition(Table tbl, Map<String, String> partSpec)
throws HiveException {
+ if (tbl.getStorageHandler() != null &&
tbl.getStorageHandler().alwaysUnpartitioned()) {
+ return tbl.getStorageHandler().getPartition(tbl, partSpec);
+ } else {
+ return getPartition(tbl, partSpec, false);
+ }
+ }
+
public Partition getPartition(Table tbl, Map<String, String> partSpec,
boolean forceCreate) throws HiveException {
return getPartition(tbl, partSpec, forceCreate, null, true);
@@ -4121,10 +4130,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.HIVE_GET_PARTITIONS);
try {
- int batchSize= MetastoreConf.getIntVar(Hive.get().getConf(),
MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX);
- return new ArrayList<>(getAllPartitionsInBatches(tbl, batchSize,
DEFAULT_BATCH_DECAYING_FACTOR, MetastoreConf
- .getIntVar(Hive.get().getConf(),
MetastoreConf.ConfVars.GETPARTITIONS_BATCH_MAX_RETRIES),
- null, true, getUserName(), getGroupNames()));
+ if (tbl.getStorageHandler() != null &&
tbl.getStorageHandler().alwaysUnpartitioned()) {
+ return tbl.getStorageHandler().getPartitions(tbl,
Collections.EMPTY_MAP);
+ } else {
+ int batchSize= MetastoreConf.getIntVar(Hive.get().getConf(),
MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX);
+ return new ArrayList<>(getAllPartitionsInBatches(tbl, batchSize,
DEFAULT_BATCH_DECAYING_FACTOR, MetastoreConf
+ .getIntVar(Hive.get().getConf(),
MetastoreConf.ConfVars.GETPARTITIONS_BATCH_MAX_RETRIES),
+ null, true, getUserName(), getGroupNames()));
+ }
} finally {
perfLogger.perfLogEnd(CLASS_NAME, PerfLogger.HIVE_GET_PARTITIONS,
"HS2-cache");
}
@@ -4282,7 +4295,11 @@ private void constructOneLBLocationMap(FileStatus fSta,
*/
public List<Partition> getPartitions(Table tbl, Map<String, String>
partialPartSpec)
throws HiveException {
- return getPartitions(tbl, partialPartSpec, (short)-1);
+ if (tbl.getStorageHandler() != null &&
tbl.getStorageHandler().alwaysUnpartitioned()) {
+ return tbl.getStorageHandler().getPartitions(tbl, partialPartSpec);
+ } else {
+ return getPartitions(tbl, partialPartSpec, (short)-1);
+ }
}
/**
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 87f0afe50b2..b76f5f9ecd2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.mapred.OutputFormat;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
/**
@@ -750,6 +751,17 @@ public interface HiveStorageHandler extends Configurable {
return true;
}
+ /**
+ * Checks if a given table and partition specifications are eligible for
compaction.
+ * @param table {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
+ * @param partitionSpec Map of Strings {@link java.util.Map} partition
specification
+ * @return Optional of ErrorMsg {@link org.apache.hadoop.hive.ql.ErrorMsg}
+ */
+ default Optional<ErrorMsg>
isEligibleForCompaction(org.apache.hadoop.hive.ql.metadata.Table table,
+ Map<String, String> partitionSpec) {
+ throw new UnsupportedOperationException("Storage handler does not support
validating eligibility for compaction");
+ }
+
default List<String>
getPartitionNames(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
Map<String, String> partitionSpec) throws SemanticException {
throw new UnsupportedOperationException("Storage handler does not support
getting partitions " +
@@ -777,6 +789,30 @@ public interface HiveStorageHandler extends Configurable {
"for a table.");
}
+ /**
+ * Returns partition based on table and partition specification.
+ * @param table {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
+ * @param partitionSpec Map of Strings {@link java.util.Map} partition
specification
+ * @return Partition {@link org.apache.hadoop.hive.ql.metadata.Partition}
+ * @throws SemanticException {@link
org.apache.hadoop.hive.ql.parse.SemanticException}
+ */
+ default Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table
table, Map<String, String> partitionSpec)
+ throws SemanticException {
+ throw new UnsupportedOperationException("Storage handler does not support
getting partition for a table.");
+ }
+
+ /**
+ * Returns a list of partitions based on table and partial partition
specification.
+ * @param table {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
+ * @param partitionSpec Map of Strings {@link java.util.Map} partition
specification
+ * @return List of Partitions {@link
org.apache.hadoop.hive.ql.metadata.Partition}
+ * @throws SemanticException {@link
org.apache.hadoop.hive.ql.parse.SemanticException}
+ */
+ default List<Partition>
getPartitions(org.apache.hadoop.hive.ql.metadata.Table table,
+ Map<String, String> partitionSpec) throws SemanticException {
+ throw new UnsupportedOperationException("Storage handler does not support
getting partitions for a table.");
+ }
+
default boolean supportsMergeFiles() {
return false;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/service/CompactionExecutorFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/service/CompactionExecutorFactory.java
index 56536d9692b..0eb5ef0780f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/service/CompactionExecutorFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/service/CompactionExecutorFactory.java
@@ -42,7 +42,7 @@ public class CompactionExecutorFactory {
Utilities.getSessionSpecifiedClassLoader());
compactionService = icebergCompactionService.newInstance();
- compactionService.init(conf, msc, compactorFactory,
collectGenericStats);
+ compactionService.init(conf, msc, compactorFactory, false);
}
catch (Exception e) {
throw new HiveException("Failed instantiating and calling Iceberg
compaction executor", e);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index e3ed4998f1c..ce04b9b8054 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1187,6 +1187,11 @@ class MetaStoreDirectSql {
MetastoreDirectSqlUtils.timingTrace(doTrace, queryText, start,
queryTime);
}
String partIds = getIdListForIn(partitions.keySet());
+
+ if (partIds.isEmpty()) {
+ return orderedResult;
+ }
+
// Now get all the one-to-many things. Start with partitions.
MetastoreDirectSqlUtils
.setPartitionParametersWithFilter(PARTITION_PARAMS,
convertMapNullsToEmptyStrings, pm,
