HIVE-14626: Support Trash in Truncate Table (Chaoyu Tang, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/943a361d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/943a361d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/943a361d Branch: refs/heads/hive-14535 Commit: 943a361da621ebe587c613c811231c606c66e515 Parents: 4c543b1 Author: ctang <[email protected]> Authored: Thu Sep 8 14:09:15 2016 -0400 Committer: ctang <[email protected]> Committed: Thu Sep 8 14:09:15 2016 -0400 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 33 ++++-- .../apache/hadoop/hive/ql/metadata/Hive.java | 2 +- .../clientpositive/encryption_drop_partition.q | 7 ++ .../clientpositive/encryption_drop_table.q | 23 ++++ .../encrypted/encryption_drop_partition.q.out | 33 ++++++ .../encrypted/encryption_drop_table.q.out | 110 +++++++++++++++++++ 6 files changed, 198 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 66cbdd3..569c19e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -225,6 +225,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.tools.HadoopArchives; @@ -4269,15 +4271,28 @@ public class DDLTask extends Task<DDLWork> implements Serializable { // this is not transactional for (Path location : getLocations(db, table, partSpec)) { FileSystem fs = location.getFileSystem(conf); - HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); - FileStatus targetStatus = fs.getFileStatus(location); - String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); - fs.delete(location, true); - fs.mkdirs(location); - try { - HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false); - } catch (Exception e) { - LOG.warn("Error setting permissions of " + location, e); + HadoopShims.HdfsEncryptionShim shim + = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); + if (!shim.isPathEncrypted(location)) { + HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); + FileStatus targetStatus = fs.getFileStatus(location); + String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); + FileUtils.moveToTrash(fs, location, conf); + fs.mkdirs(location); + try { + HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false); + } catch (Exception e) { + LOG.warn("Error setting permissions of " + location, e); + } + } else { + FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER); + if (statuses == null || statuses.length == 0) { + continue; + } + boolean success = Hive.trashFiles(fs, statuses, conf); + if (!success) { + throw new HiveException("Error in deleting the contents of " + location.toString()); + } } } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 5f53aef..da46854 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -3365,7 +3365,7 @@ private void constructOneLBLocationMap(FileStatus fSta, * @return true if deletion successful * @throws IOException */ - private boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf) + public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf) throws IOException { boolean result = true; http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/queries/clientpositive/encryption_drop_partition.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q index 57dfabd..d968459 100644 --- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q +++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q @@ -25,4 +25,11 @@ ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23'); SELECT * FROM encrypted_table_dp; ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE; SELECT * FROM encrypted_table_dp; + +TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24'); +SHOW PARTITIONS encrypted_table_dp; +SELECT * FROM encrypted_table_dp; + +ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24'); +DROP TABLE encrypted_table_dp; DROP TABLE encrypted_table_dp PURGE; http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/queries/clientpositive/encryption_drop_table.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q index 2ae3c69..76be118 100644 --- a/ql/src/test/queries/clientpositive/encryption_drop_table.q +++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q @@ -5,6 +5,8 @@ set hive.cli.errors.ignore=true; DROP TABLE IF EXISTS encrypted_table PURGE; +DROP TABLE IF EXISTS encrypted_ext_table PURGE; + CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; @@ -22,4 +24,25 @@ SHOW TABLES; DROP TABLE default.encrypted_table PURGE; SHOW TABLES; + +DROP TABLE IF EXISTS encrypted_table1 PURGE; +CREATE TABLE encrypted_table1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table1'; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table1; +INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src; + +SELECT COUNT(*) FROM encrypted_table1; +TRUNCATE TABLE encrypted_table1; +SELECT COUNT(*) FROM encrypted_table1; + +INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src; +DROP TABLE default.encrypted_table1; +SHOW TABLES; + +TRUNCATE TABLE encrypted_table1; +DROP TABLE default.encrypted_table1; +SHOW TABLES; + +DROP TABLE default.encrypted_table1 PURGE; +SHOW TABLES; + CRYPTO DELETE_KEY --keyName key_128; http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out index 2643006..c2f6b03 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out @@ -147,6 +147,39 @@ POSTHOOK: Input: default@encrypted_table_dp POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24 #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging 2 bar 2014-09-24 +PREHOOK: query: TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24') +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@encrypted_table_dp@p=2014-09-24 +POSTHOOK: query: TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24') +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@encrypted_table_dp@p=2014-09-24 +PREHOOK: query: SHOW PARTITIONS encrypted_table_dp +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@encrypted_table_dp +POSTHOOK: query: SHOW PARTITIONS encrypted_table_dp +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@encrypted_table_dp +p=2014-09-24 +PREHOOK: query: SELECT * FROM encrypted_table_dp +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table_dp +PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_table_dp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table_dp +POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@encrypted_table_dp +PREHOOK: Output: default@encrypted_table_dp@p=2014-09-24 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to drop default.encrypted_table_dp.[2014-09-24] because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash. +PREHOOK: query: DROP TABLE encrypted_table_dp +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table_dp +PREHOOK: Output: default@encrypted_table_dp +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table_dp because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) PREHOOK: query: DROP TABLE encrypted_table_dp PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table_dp http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out index c5007ee..a3f41d8 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out @@ -2,6 +2,10 @@ PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE #### A masked pattern was here #### @@ -87,3 +91,109 @@ POSTHOOK: query: SHOW TABLES POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default src +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PURGE +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table1 +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table1 +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table1' using key: 'key_128' +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table1 +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table1 +POSTHOOK: Lineage: encrypted_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) FROM encrypted_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table1 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging +POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table1 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging +500 +PREHOOK: query: TRUNCATE TABLE encrypted_table1 +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@encrypted_table1 +POSTHOOK: query: TRUNCATE TABLE encrypted_table1 +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@encrypted_table1 +PREHOOK: query: SELECT COUNT(*) FROM encrypted_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table1 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging +POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table1 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging +0 +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table1 +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table1 +POSTHOOK: Lineage: encrypted_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DROP TABLE default.encrypted_table1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table1 +PREHOOK: Output: default@encrypted_table1 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table1 because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +encrypted_table1 +src +PREHOOK: query: TRUNCATE TABLE encrypted_table1 +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@encrypted_table1 +POSTHOOK: query: TRUNCATE TABLE encrypted_table1 +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@encrypted_table1 +PREHOOK: query: DROP TABLE default.encrypted_table1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table1 +PREHOOK: Output: default@encrypted_table1 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table1 because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +encrypted_table1 +src +PREHOOK: query: DROP TABLE default.encrypted_table1 PURGE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table1 +PREHOOK: Output: default@encrypted_table1 +POSTHOOK: query: DROP TABLE default.encrypted_table1 PURGE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table1 +POSTHOOK: Output: default@encrypted_table1 +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +src
