Repository: hive Updated Branches: refs/heads/branch-3 51e6f524e -> 434754c18
HIVE-20329: Long running repl load (incr/bootstrap) causing OOM error (Mahesh Kumar Behera, reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/434754c1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/434754c1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/434754c1 Branch: refs/heads/branch-3 Commit: 434754c1849393fb42b92359703dde408be52b46 Parents: 51e6f52 Author: Sankar Hariappan <[email protected]> Authored: Mon Aug 13 11:25:39 2018 +0530 Committer: Sankar Hariappan <[email protected]> Committed: Mon Aug 13 11:25:39 2018 +0530 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java | 6 ++++++ .../apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java | 11 +++++++++++ 2 files changed, 17 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/434754c1/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 09e7ba1..ffbe3bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -72,6 +72,12 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable { @Override protected int execute(DriverContext driverContext) { + Task<? extends Serializable> rootTask = work.getRootTask(); + if (rootTask != null) { + rootTask.setChildTasks(null); + } + work.setRootTask(this); + this.parentTasks = null; if (work.isIncrementalLoad()) { return executeIncrementalLoad(driverContext); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/434754c1/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java index 8921e94..fdbcb15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadEventsIter import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadTasksBuilder; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.session.LineageState; +import org.apache.hadoop.hive.ql.exec.Task; import java.io.IOException; import java.io.Serializable; @@ -42,6 +43,7 @@ public class ReplLoadWork implements Serializable { private int loadTaskRunCount = 0; private DatabaseEvent.State state = null; private final transient IncrementalLoadTasksBuilder incrementalLoad; + private transient Task<? extends Serializable> rootTask; /* these are sessionState objects that are copied over to work to allow for parallel execution. @@ -56,6 +58,7 @@ public class ReplLoadWork implements Serializable { sessionStateLineageState = lineageState; this.dumpDirectory = dumpDirectory; this.dbNameToLoadIn = dbNameToLoadIn; + rootTask = null; if (isIncrementalDump) { incrementalIterator = new IncrementalLoadEventsIterator(dumpDirectory, hiveConf); this.bootstrapIterator = null; @@ -110,4 +113,12 @@ public class ReplLoadWork implements Serializable { public IncrementalLoadTasksBuilder getIncrementalLoadTaskBuilder() { return incrementalLoad; } + + public Task<? extends Serializable> getRootTask() { + return rootTask; + } + + public void setRootTask(Task<? extends Serializable> rootTask) { + this.rootTask = rootTask; + } }
