HIVE-9811 : Hive on Tez leaks WorkMap objects (Oleg Danilov, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3672a279 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3672a279 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3672a279 Branch: refs/heads/beeline-cli Commit: 3672a279a6fa46fa2a55346ef2257cf52a9900da Parents: 4c0fb13 Author: Sergey Shelukhin <[email protected]> Authored: Fri Sep 18 15:01:04 2015 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Fri Sep 18 15:01:04 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3672a279/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 3a6ec1a..4a1a712 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -83,6 +83,9 @@ public class TezTask extends Task<TezWork> { private final DagUtils utils; + Map<BaseWork, Vertex> workToVertex = new HashMap<BaseWork, Vertex>(); + Map<BaseWork, JobConf> workToConf = new HashMap<BaseWork, JobConf>(); + public TezTask() { this(DagUtils.getInstance()); } @@ -197,6 +200,15 @@ public class TezTask extends Task<TezWork> { // rc will be 1 at this point indicating failure. } finally { Utilities.clearWork(conf); + + // Clear gWorkMap + for (BaseWork w : work.getAllWork()) { + JobConf workCfg = workToConf.get(w); + if (workCfg != null) { + Utilities.clearWorkMapForConf(workCfg); + } + } + if (cleanContext) { try { ctx.clear(); @@ -276,8 +288,6 @@ public class TezTask extends Task<TezWork> { throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_BUILD_DAG); - Map<BaseWork, Vertex> workToVertex = new HashMap<BaseWork, Vertex>(); - Map<BaseWork, JobConf> workToConf = new HashMap<BaseWork, JobConf>(); // getAllWork returns a topologically sorted list, which we use to make // sure that vertices are created before they are used in edges.
