[hive] branch master updated: HIVE-22744 : TezTask for the vertex with more than one outedge should have proportional sort memory (Ramesh Kumar via Ashutosh Chauhan)

[hashutosh]

This is an automated email from the ASF dual-hosted git repository.

hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new c3f3523  HIVE-22744 : TezTask for the vertex with more than one 
outedge should have proportional sort memory (Ramesh Kumar via Ashutosh Chauhan)
c3f3523 is described below

commit c3f3523b498748f4a309df807893e0a4f94c1338
Author: Ramesh Kumar Thangarajan <rameshku...@cloudera.com>
AuthorDate: Fri Feb 21 08:38:05 2020 -0800

    HIVE-22744 : TezTask for the vertex with more than one outedge should have 
proportional sort memory (Ramesh Kumar via Ashutosh Chauhan)
    
    Signed-off-by: Ashutosh Chauhan <hashut...@apache.org>
---
 ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 4002ccb..0bacb05 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -82,6 +82,7 @@ import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.StatusGetOpts;
 import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -479,6 +480,18 @@ public class TezTask extends Task<TezWork> {
         checkOutputSpec(w, wxConf);
         Vertex wx = utils.createVertex(wxConf, w, scratchDir, fs, ctx, 
!isFinal,
             work, work.getVertexType(w), vertexResources);
+        if (work.getChildren(w).size() > 1) {
+          String value = 
wxConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB);
+          int originalValue = 0;
+          if(value == null) {
+            originalValue = 
TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB_DEFAULT;
+          } else {
+            originalValue = Integer.valueOf(value);
+          }
+          int newValue = (int) (originalValue / work.getChildren(w).size());
+          wxConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 
Integer.toString(newValue));
+          LOG.info("Modified " + 
TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + " to " + newValue);
+        }
         if (w.getReservedMemoryMB() > 0) {
           // If reversedMemoryMB is set, make memory allocation fraction 
adjustment as needed
           double frac = 
DagUtils.adjustMemoryReserveFraction(w.getReservedMemoryMB(), super.conf);