Author: rohini
Date: Thu Apr 16 14:46:56 2015
New Revision: 1674079
URL: http://svn.apache.org/r1674079
Log:
PIG-4505: [Pig on Tez] Auto adjust AM memory can hit OOM with 3.5GXmx
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1674079&r1=1674078&r2=1674079&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Apr 16 14:46:56 2015
@@ -62,6 +62,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4505: [Pig on Tez] Auto adjust AM memory can hit OOM with 3.5GXmx (rohini)
+
PIG-4502: E2E tests build fail with udfs compile (nmaheshwari via daijy)
PIG-4498: AvroStorage in Piggbank does not handle bad records and fails (viraj
via rohini)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java?rev=1674079&r1=1674078&r2=1674079&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
Thu Apr 16 14:46:56 2015
@@ -104,28 +104,37 @@ public class TezSessionManager {
private static void adjustAMConfig(TezConfiguration amConf, TezJobConfig
tezJobConf) {
int requiredAMMaxHeap = -1;
int requiredAMResourceMB = -1;
- int configuredAMMaxHeap = Utils.extractHeapSizeInMB(amConf.get(
+ String amLaunchOpts = amConf.get(
TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
- TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT));
+ TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);
+ int configuredAMMaxHeap = Utils.extractHeapSizeInMB(amLaunchOpts);
int configuredAMResourceMB = amConf.getInt(
TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB,
TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT);
if (tezJobConf.getEstimatedTotalParallelism() > 0) {
- int minAMMaxHeap = 3584;
+ // Need more room for native memory/virtual address space
+ // when close to 4G due to 32-bit jvm 4G limit
+ int minAMMaxHeap = 3200;
int minAMResourceMB = 4096;
// Rough estimation. For 5K tasks 1G Xmx and 1.5G resource.mb
- // Increment by 512 mb for every additional 5K tasks.
+ // Increment container size by 512 mb for every additional 5K
tasks.
+ // 30000 and above - 3200Xmx, 4096 (896 native memory)
+ // 25000 and above - 3072Xmx, 3584
+ // 20000 and above - 2560Xmx, 3072
+ // 15000 and above - 2048Xmx, 2560
+ // 10000 and above - 1536Xmx, 2048
+ // 5000 and above - 1024Xmx, 1536 (512 native memory)
for (int taskCount = 30000; taskCount >= 5000; taskCount-=5000) {
- if (tezJobConf.getEstimatedTotalParallelism() > taskCount) {
+ if (tezJobConf.getEstimatedTotalParallelism() >= taskCount) {
requiredAMMaxHeap = minAMMaxHeap;
requiredAMResourceMB = minAMResourceMB;
break;
}
- minAMMaxHeap = minAMMaxHeap - 512;
minAMResourceMB = minAMResourceMB - 512;
+ minAMMaxHeap = minAMResourceMB - 512;
}
if (requiredAMResourceMB > -1 && configuredAMResourceMB <
requiredAMResourceMB) {
@@ -139,13 +148,14 @@ public class TezSessionManager {
if (requiredAMMaxHeap > -1 && configuredAMMaxHeap <
requiredAMMaxHeap) {
amConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
- amConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS)
- + " -Xmx" + requiredAMMaxHeap + "M");
+ amLaunchOpts + " -Xmx" + requiredAMMaxHeap + "M");
log.info("Increasing Tez AM Heap Size from "
+ configuredAMMaxHeap + "M to "
+ requiredAMMaxHeap
+ "M as the number of total estimated tasks is "
+ tezJobConf.getEstimatedTotalParallelism());
+ log.info("Value of " +
TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS + " is now "
+ +
amConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS));
}
}
}
