hive git commit: HIVE-20648: LLAP: Vector group by operator should use memory per executor

2018-10-09 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/branch-3 b3a424bd0 -> 8703a3229


HIVE-20648: LLAP: Vector group by operator should use memory per executor


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8703a322
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8703a322
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8703a322

Branch: refs/heads/branch-3
Commit: 8703a3229e5a8d4afdd0e2ebd06579df40b01ed8
Parents: b3a424b
Author: Prasanth Jayachandran 
Authored: Tue Oct 9 11:05:55 2018 -0700
Committer: Prasanth Jayachandran 
Committed: Tue Oct 9 11:06:28 2018 -0700

--
 .../ql/exec/vector/VectorGroupByOperator.java   | 24 +++--
 .../exec/vector/TestVectorGroupByOperator.java  | 96 
 2 files changed, 112 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8703a322/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
index 43f1162..4dfd179 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
@@ -33,6 +33,8 @@ import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.LlapUtil;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.ql.CompilationOpContext;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.IConfigureJobConf;
@@ -146,6 +148,7 @@ public class VectorGroupByOperator extends 
Operator
 
   private float memoryThreshold;
 
+  private boolean isLlap = false;
   /**
* Interface for processing mode: global, hash, unsorted streaming, or group 
batch
*/
@@ -515,7 +518,7 @@ public class VectorGroupByOperator extends 
Operator
   aggregationBatchInfo.getAggregatorsFixedSize();
 
   MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
-  maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
+  maxMemory = isLlap ? getConf().getMaxMemoryAvailable() : 
memoryMXBean.getHeapMemoryUsage().getMax();
   memoryThreshold = conf.getMemoryThreshold();
   // Tests may leave this unitialized, so better set it to 1
   if (memoryThreshold == 0.0f) {
@@ -525,13 +528,14 @@ public class VectorGroupByOperator extends 
Operator
   maxHashTblMemory = (int)(maxMemory * memoryThreshold);
 
   if (LOG.isDebugEnabled()) {
-LOG.debug(String.format("maxMemory:%dMb (%d * %f) fixSize:%d (key:%d 
agg:%d)",
-maxHashTblMemory/1024/1024,
-maxMemory/1024/1024,
-memoryThreshold,
-fixedHashEntrySize,
-keyWrappersBatch.getKeysFixedSize(),
-aggregationBatchInfo.getAggregatorsFixedSize()));
+LOG.debug("GBY memory limits - isLlap: {} maxMemory: {} ({} * {}) 
fixSize:{} (key:{} agg:{})",
+  isLlap,
+  LlapUtil.humanReadableByteCount(maxHashTblMemory),
+  LlapUtil.humanReadableByteCount(maxMemory),
+  memoryThreshold,
+  fixedHashEntrySize,
+  keyWrappersBatch.getKeysFixedSize(),
+  aggregationBatchInfo.getAggregatorsFixedSize());
   }
 }
 
@@ -975,6 +979,7 @@ public class VectorGroupByOperator extends 
Operator
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
 super.initializeOp(hconf);
+isLlap = LlapProxy.isDaemon();
 VectorExpression.doTransientInit(keyExpressions);
 
 List objectInspectors = new ArrayList();
@@ -1231,4 +1236,7 @@ public class VectorGroupByOperator extends 
Operator
 }
   }
 
+  public long getMaxMemory() {
+return maxMemory;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/8703a322/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
index ffdc410..e2a593f 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
@@ -38,6 +38,8 @@ import java.util.Set;
 
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.conf.HiveConf;

hive git commit: HIVE-20648: LLAP: Vector group by operator should use memory per executor

2018-10-09 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/master db04f3f9a -> 2d2ab674f


HIVE-20648: LLAP: Vector group by operator should use memory per executor


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d2ab674
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d2ab674
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d2ab674

Branch: refs/heads/master
Commit: 2d2ab674f8acb8a4e1d0532790e6c27bd8553018
Parents: db04f3f
Author: Prasanth Jayachandran 
Authored: Tue Oct 9 11:05:55 2018 -0700
Committer: Prasanth Jayachandran 
Committed: Tue Oct 9 11:05:55 2018 -0700

--
 .../ql/exec/vector/VectorGroupByOperator.java   | 24 +++--
 .../exec/vector/TestVectorGroupByOperator.java  | 96 
 2 files changed, 112 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/2d2ab674/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
index a516d60..0d80c9e 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
@@ -33,6 +33,8 @@ import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.LlapUtil;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.ql.CompilationOpContext;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.IConfigureJobConf;
@@ -148,6 +150,7 @@ public class VectorGroupByOperator extends 
Operator
 
   private float memoryThreshold;
 
+  private boolean isLlap = false;
   /**
* Interface for processing mode: global, hash, unsorted streaming, or group 
batch
*/
@@ -517,7 +520,7 @@ public class VectorGroupByOperator extends 
Operator
   aggregationBatchInfo.getAggregatorsFixedSize();
 
   MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
-  maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
+  maxMemory = isLlap ? getConf().getMaxMemoryAvailable() : 
memoryMXBean.getHeapMemoryUsage().getMax();
   memoryThreshold = conf.getMemoryThreshold();
   // Tests may leave this unitialized, so better set it to 1
   if (memoryThreshold == 0.0f) {
@@ -527,13 +530,14 @@ public class VectorGroupByOperator extends 
Operator
   maxHashTblMemory = (int)(maxMemory * memoryThreshold);
 
   if (LOG.isDebugEnabled()) {
-LOG.debug(String.format("maxMemory:%dMb (%d * %f) fixSize:%d (key:%d 
agg:%d)",
-maxHashTblMemory/1024/1024,
-maxMemory/1024/1024,
-memoryThreshold,
-fixedHashEntrySize,
-keyWrappersBatch.getKeysFixedSize(),
-aggregationBatchInfo.getAggregatorsFixedSize()));
+LOG.debug("GBY memory limits - isLlap: {} maxMemory: {} ({} * {}) 
fixSize:{} (key:{} agg:{})",
+  isLlap,
+  LlapUtil.humanReadableByteCount(maxHashTblMemory),
+  LlapUtil.humanReadableByteCount(maxMemory),
+  memoryThreshold,
+  fixedHashEntrySize,
+  keyWrappersBatch.getKeysFixedSize(),
+  aggregationBatchInfo.getAggregatorsFixedSize());
   }
 }
 
@@ -977,6 +981,7 @@ public class VectorGroupByOperator extends 
Operator
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
 super.initializeOp(hconf);
+isLlap = LlapProxy.isDaemon();
 VectorExpression.doTransientInit(keyExpressions);
 
 List objectInspectors = new ArrayList();
@@ -1233,4 +1238,7 @@ public class VectorGroupByOperator extends 
Operator
 }
   }
 
+  public long getMaxMemory() {
+return maxMemory;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d2ab674/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
index fe1375b..278f167 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
@@ -38,6 +38,8 @@ import java.util.Set;
 
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.conf.HiveConf;