[hive] branch master updated: HIVE-24235: Drop and recreate table during MR compaction leaves behind base/delta directory (Karen Coppage, reviewed by Peter Vary)

[klcopp]

This is an automated email from the ASF dual-hosted git repository.

klcopp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 75d6057  HIVE-24235: Drop and recreate table during MR compaction 
leaves behind base/delta directory (Karen Coppage, reviewed by Peter Vary)
75d6057 is described below

commit 75d6057abc214bf32a09cbd26809dad713457ff0
Author: Karen Coppage <karenlcopp...@gmail.com>
AuthorDate: Thu Oct 15 10:15:47 2020 +0200

    HIVE-24235: Drop and recreate table during MR compaction leaves behind 
base/delta directory (Karen Coppage, reviewed by Peter Vary)
    
    Closes #1558
---
 .../org/apache/hadoop/hive/ql/txn/compactor/Worker.java    | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 605821b..572d003 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.TxnType;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hive.common.util.Ref;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
@@ -141,6 +142,16 @@ public class Worker extends RemoteCompactorThread 
implements MetaStoreThread {
     }
   }
 
+  private void verifyTableIdHasNotChanged(CompactionInfo ci, Table 
originalTable) throws HiveException, MetaException {
+    Table currentTable = resolveTable(ci);
+    if (originalTable.getId() != currentTable.getId()) {
+      throw new HiveException("Table " + originalTable.getDbName() + "." + 
originalTable.getTableName()
+          + " id (" + currentTable.getId() + ") is not equal to its id when 
compaction started ("
+          + originalTable.getId() + "). The table might have been dropped and 
recreated while compaction was running."
+          + " Marking compaction as failed.");
+    }
+  }
+
   private void commitTxnIfSet(long compactorTxnId) {
     if (compactorTxnId != TXN_ID_NOT_SET) {
       try {
@@ -540,6 +551,9 @@ public class Worker extends RemoteCompactorThread 
implements MetaStoreThread {
           }
         }
         heartbeater.cancel();
+
+        verifyTableIdHasNotChanged(ci, t1);
+
         LOG.info("Completed " + ci.type.toString() + " compaction for " + 
ci.getFullPartitionName() + " in txn "
             + JavaUtils.txnIdToString(compactorTxnId) + ", marking as 
compacted.");
         msc.markCompacted(CompactionInfo.compactionInfoToStruct(ci));