Repository: hive Updated Branches: refs/heads/master 250e10ecf -> 681051f10
HIVE-20329: Long running repl load (incr/bootstrap) causing OOM error (Mahesh Kumar Behera, reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/681051f1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/681051f1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/681051f1 Branch: refs/heads/master Commit: 681051f1080ad5eed4ac8a8e5daf187ff23d4839 Parents: 250e10e Author: Sankar Hariappan <[email protected]> Authored: Mon Aug 13 11:24:28 2018 +0530 Committer: Sankar Hariappan <[email protected]> Committed: Mon Aug 13 11:24:28 2018 +0530 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java | 6 ++++++ .../apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java | 11 +++++++++++ 2 files changed, 17 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/681051f1/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 6b5fc54..aa41301 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -72,6 +72,12 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable { @Override protected int execute(DriverContext driverContext) { + Task<? extends Serializable> rootTask = work.getRootTask(); + if (rootTask != null) { + rootTask.setChildTasks(null); + } + work.setRootTask(this); + this.parentTasks = null; if (work.isIncrementalLoad()) { return executeIncrementalLoad(driverContext); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/681051f1/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java index 8921e94..fdbcb15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadEventsIter import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadTasksBuilder; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.session.LineageState; +import org.apache.hadoop.hive.ql.exec.Task; import java.io.IOException; import java.io.Serializable; @@ -42,6 +43,7 @@ public class ReplLoadWork implements Serializable { private int loadTaskRunCount = 0; private DatabaseEvent.State state = null; private final transient IncrementalLoadTasksBuilder incrementalLoad; + private transient Task<? extends Serializable> rootTask; /* these are sessionState objects that are copied over to work to allow for parallel execution. @@ -56,6 +58,7 @@ public class ReplLoadWork implements Serializable { sessionStateLineageState = lineageState; this.dumpDirectory = dumpDirectory; this.dbNameToLoadIn = dbNameToLoadIn; + rootTask = null; if (isIncrementalDump) { incrementalIterator = new IncrementalLoadEventsIterator(dumpDirectory, hiveConf); this.bootstrapIterator = null; @@ -110,4 +113,12 @@ public class ReplLoadWork implements Serializable { public IncrementalLoadTasksBuilder getIncrementalLoadTaskBuilder() { return incrementalLoad; } + + public Task<? extends Serializable> getRootTask() { + return rootTask; + } + + public void setRootTask(Task<? extends Serializable> rootTask) { + this.rootTask = rootTask; + } }
