dengzhhu653 commented on code in PR #6935:
URL: https://github.com/apache/hive/pull/6935#discussion_r4520866304
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/TruncateTableHandler.java:
##########
@@ -142,112 +124,162 @@ protected TruncateTableResult execute() throws
TException, IOException {
.map(Boolean::parseBoolean)
.orElse(false);
- if (TxnUtils.isTransactionalTable(table) || !skipDataDeletion) {
- if (!skipDataDeletion) {
- isSkipTrash = MetaStoreUtils.isSkipTrash(table.getParameters());
- Database db = handler.get_database_core(catName, dbName);
- needCmRecycle = ReplChangeManager.shouldEnableCm(db, table);
- }
- List<Path> locations = new ArrayList<>();
- if (0 != table.getPartitionKeysSize()) {
- for (Partition partition : partitions) {
- locations.add(new Path(partition.getSd().getLocation()));
- }
- } else {
- locations.add(new Path(table.getSd().getLocation()));
- }
- // This is not transactional
- for (Path location : locations) {
- if (!skipDataDeletion) {
- truncateDataFiles(location, isSkipTrash, needCmRecycle);
- } else {
- // For Acid tables we don't need to delete the old files, only write
an empty baseDir.
- // Compaction and cleaner will take care of the rest
- addTruncateBaseFile(location, request.getWriteId(),
- handler.getConf(), AcidMetaDataFile.DataFormat.TRUNCATED);
- }
- }
+ if (!skipDataDeletion) {
+ isSkipTrash = MetaStoreUtils.isSkipTrash(table.getParameters());
+ Database db = handler.get_database_core(catName, dbName);
+ needCmRecycle = ReplChangeManager.shouldEnableCm(db, table);
+ }
+
+ if (skipDataDeletion && !isTxnTable) {
+ throw new MetaException("Cannot truncate the table as request configured
to skip deleting data" +
+ " and the target isn't a transactional table");
}
- // Alter the table/partition stats and also notify truncate table event
+
boolean success;
if (isPartitioned) {
- success = alterPartitionsForTruncate();
+ success = truncatePartitions(isSkipTrash, needCmRecycle,
skipDataDeletion);
} else {
+ truncateLocation(new Path(table.getSd().getLocation()), isSkipTrash,
needCmRecycle, skipDataDeletion);
success = alterTableStatsForTruncate();
}
return new TruncateTableResult(success);
}
- private void updateStatsForTruncate(Map<String,String> props,
EnvironmentContext environmentContext) {
- if (null == props) {
- return;
+ private PartitionsIterable newPartitionBatches(int batchSize) throws
MetaException {
+ TableName qualifiedTable = new TableName(catName, dbName,
request.getTableName());
+ if (reqPartNames != null) {
+ return PartitionsIterable.create(
+ ms.unwrap(TableStore.class), qualifiedTable, batchSize,
reqPartNames);
}
- for (String stat : StatsSetupConst.SUPPORTED_STATS) {
- String statVal = props.get(stat);
- if (statVal != null) {
- //In the case of truncate table, we set the stats to be 0.
- props.put(stat, "0");
+ return PartitionsIterable.create(ms, qualifiedTable, batchSize);
+ }
+
+ private boolean truncatePartitions(boolean isSkipTrash, boolean
needCmRecycle, boolean skipDataDeletion)
+ throws TException, IOException {
+ if (reqPartNames != null && reqPartNames.isEmpty()) {
+ return true;
+ }
+ int batchSize = MetastoreConf.getIntVar(handler.getConf(),
+ MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX);
+ PartitionsIterable partitionBatches = newPartitionBatches(batchSize);
+
+ boolean success = true;
+ for (List<Partition> partitions : partitionBatches) {
+ if (partitions.isEmpty()) {
+ continue;
+ }
+ EnvironmentContext environmentContext = new EnvironmentContext();
+ List<Partition> newPartitions = buildAndCheckPartitions(partitions,
environmentContext);
Review Comment:
It's intentional. A validation pass beforehand would read every partition
once to fire the pre-event, then read them again to delete and alter. That
doubles the metastore reads on every partitioned truncate, which is the cost
this change avoids. The truncate table is a best-effort operation, i.e, we
cannot restore the deleted data if the metadata change fails.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
