Repository: tez Updated Branches: refs/heads/master 11b584318 -> 19378d5c2
EZ-1482. Fix memory issues for Local Mode running concurrent tasks (pramachandran) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19378d5c Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19378d5c Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19378d5c Branch: refs/heads/master Commit: 19378d5c2d79006b19140f2c5b9740d04942a3b8 Parents: 11b5843 Author: Prakash Ramachandran <[email protected]> Authored: Wed Apr 15 21:48:07 2015 +0530 Committer: Prakash Ramachandran <[email protected]> Committed: Wed Apr 15 21:48:07 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../tez/dag/app/launcher/LocalContainerLauncher.java | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 605577e..56f6ea9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly ALL CHANGES: + TEZ-1482. Fix memory issues for Local Mode running concurrent tasks TEZ-2033. Update TestOrderedWordCount to add processor configs as history text and use MR configs correctly TEZ-2318. Tez UI: source and sink page is broken as they are not populated. http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java index 7f5a7b1..5c8aab6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java @@ -90,6 +90,7 @@ public class LocalContainerLauncher extends AbstractService implements private final String workingDirectory; private final Map<String, String> localEnv = new HashMap<String, String>(); private final ExecutionContext executionContext; + private int numExecutors; private final ConcurrentHashMap<ContainerId, ListenableFuture<TezChild.ContainerExecutionResult>> runningContainers = @@ -122,7 +123,7 @@ public class LocalContainerLauncher extends AbstractService implements @Override public synchronized void serviceInit(Configuration conf) { - int numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS, + numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS, TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT); Preconditions.checkState(numExecutors >=1, "Must have at least 1 executor"); ExecutorService rawExecutor = Executors.newFixedThreadPool(numExecutors, @@ -328,11 +329,14 @@ public class LocalContainerLauncher extends AbstractService implements containerEnv.putAll(localEnv); containerEnv.put(Environment.USER.name(), context.getUser()); - // TODO TEZ-1482. Control the memory available based on number of executors + long memAvailable; + synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning + memAvailable = Runtime.getRuntime().maxMemory() / numExecutors; + } TezChild tezChild = TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier, attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext, credentials, - Runtime.getRuntime().maxMemory(), context.getUser()); + memAvailable, context.getUser()); tezChild.setUmbilical(tezTaskUmbilicalProtocol); return tezChild; }
