HIVE-20905 : querying streaming table fails with out of memory exception 
(mahesh kumar behera via Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/148e7acb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/148e7acb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/148e7acb

Branch: refs/heads/master-tez092
Commit: 148e7acba46da997a023b57794c7f1f209097320
Parents: 52f94b8
Author: Mahesh Kumar Behera <mbeh...@hortonworks.com>
Authored: Tue Nov 13 14:03:23 2018 -0700
Committer: Thejas M Nair <the...@hortonworks.com>
Committed: Tue Nov 13 14:03:23 2018 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/txn/compactor/Cleaner.java    | 12 +++++++-----
 .../org/apache/hadoop/hive/metastore/HiveMetaStore.java |  3 +++
 2 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/148e7acb/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 3565616..3bc1f8a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.hive.ql.txn.compactor;
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.metastore.ReplChangeManager;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.fs.FileStatus;
@@ -58,6 +57,8 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static 
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+
 /**
  * A class to clean directories after compactions.  This will run in a 
separate thread.
  */
@@ -323,7 +324,7 @@ public class Cleaner extends CompactorThread {
     return " id=" + ci.id;
   }
   private void removeFiles(String location, ValidWriteIdList writeIdList, 
CompactionInfo ci)
-          throws IOException, HiveException {
+          throws IOException, NoSuchObjectException {
     Path locPath = new Path(location);
     AcidUtils.Directory dir = AcidUtils.getAcidState(locPath, conf, 
writeIdList);
     List<FileStatus> obsoleteDirs = dir.getObsolete();
@@ -349,11 +350,12 @@ public class Cleaner extends CompactorThread {
     }
 
     FileSystem fs = filesToDelete.get(0).getFileSystem(conf);
-    Database db = Hive.get().getDatabase(ci.dbname);
+    Database db = rs.getDatabase(getDefaultCatalog(conf), ci.dbname);
+    Boolean isSourceOfRepl = ReplChangeManager.isSourceOfReplication(db);
 
     for (Path dead : filesToDelete) {
       LOG.debug("Going to delete path " + dead.toString());
-      if (ReplChangeManager.isSourceOfReplication(db)) {
+      if (isSourceOfRepl) {
         replChangeManager.recycle(dead, ReplChangeManager.RecycleType.MOVE, 
true);
       }
       fs.delete(dead, true);

http://git-wip-us.apache.org/repos/asf/hive/blob/148e7acb/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 0485184..23a78ca 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -9085,6 +9085,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
         conf.set((String) item.getKey(), (String) item.getValue());
       }
 
+      //for metastore process, all metastore call should be embedded metastore 
call.
+      conf.set(ConfVars.THRIFT_URIS.getHiveName(), "");
+
       // Add shutdown hook.
       shutdownHookMgr.addShutdownHook(() -> {
         String shutdownMsg = "Shutting down hive metastore.";

Reply via email to