Repository: tez Updated Branches: refs/heads/master dbd763fd4 -> a8c930481
TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures. (Greg Senia via hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fac2f5fe Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fac2f5fe Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fac2f5fe Branch: refs/heads/master Commit: fac2f5fe1e84ed57db045d0467529b3627b91458 Parents: dbd763f Author: Hitesh Shah <[email protected]> Authored: Mon Mar 14 11:17:51 2016 -0700 Committer: Hitesh Shah <[email protected]> Committed: Mon Mar 14 11:17:51 2016 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../tez/util/TezMxBeanResourceCalculator.java | 65 ++++++++++++++++++-- 2 files changed, 60 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/fac2f5fe/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 3c72884..e402071 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures. TEZ-3155. Support a way to submit DAGs to a session where the DAG plan exceeds hadoop ipc limits. TEZ-2863. Container, node, and logs not available in UI for tasks that fail to launch TEZ-3140. Reduce AM memory usage during serialization http://git-wip-us.apache.org/repos/asf/tez/blob/fac2f5fe/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java index 1f8e922..07f34e6 100644 --- a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java +++ b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java @@ -22,14 +22,19 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import java.lang.management.ManagementFactory; import java.util.concurrent.TimeUnit; +import java.lang.management.OperatingSystemMXBean; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; /** - * Uses sun's MBeans to return process information. + * Uses Sun/Oracle or IBM MBeans to return process information. */ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { - private final com.sun.management.OperatingSystemMXBean osBean; + private final OperatingSystemMXBean osBean; private final Runtime runtime; + private static final Method getCommittedVirtualMemorySize = getMxBeanMethod("getCommittedVirtualMemorySize"); + private static final Method getProcessCpuTime = getMxBeanMethod("getProcessCpuTime"); /** * Create process-tree instance with specified root process. @@ -41,8 +46,8 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { public TezMxBeanResourceCalculator(String root) { super(root); runtime = Runtime.getRuntime(); - osBean = - (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + osBean = ManagementFactory.getOperatingSystemMXBean(); + } @Override public void updateProcessTree() { @@ -54,7 +59,15 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { } @Override public long getCumulativeVmem(int olderThanAge) { - return osBean.getCommittedVirtualMemorySize(); + try { + return (Long) getCommittedVirtualMemorySize.invoke(osBean); + } catch (IllegalArgumentException e) { + return -1; + } catch (IllegalAccessException e) { + return -1; + } catch (InvocationTargetException e) { + return -1; + } } @Override public long getCumulativeRssmem(int olderThanAge) { @@ -64,7 +77,16 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { @Override public long getCumulativeCpuTime() { //convert to milliseconds - return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS); + try { + return TimeUnit.MILLISECONDS.convert( + (Long) getProcessCpuTime.invoke(osBean), TimeUnit.NANOSECONDS); + } catch (InvocationTargetException e) { + return -1; + } catch (IllegalArgumentException e) { + return -1; + } catch (IllegalAccessException e) { + return -1; + } } @Override public boolean checkPidPgrpidForMatch() { @@ -76,4 +98,35 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { //Returning -1 to indicate, this feature is not yet supported. return -1; } + + private static Method getMxBeanMethod(String methodName) { + // New Method to support IBM and Oracle/OpenJDK JDK with OperatingSystemMXBean + final String JAVA_VENDOR_NAME = System.getProperty("java.vendor"); + final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM"); + try { + final Class<?> mbeanClazz; + if (IBM_JAVA) { + mbeanClazz = Class.forName("com.ibm.lang.management.OperatingSystemMXBean"); + } else { + mbeanClazz = Class.forName("com.sun.management.OperatingSystemMXBean"); + } + if (IBM_JAVA){ + if (methodName.equals("getCommittedVirtualMemorySize")) { + methodName = "getProcessVirtualMemorySize"; + } + if (methodName.equals("getProcessCpuTime")) { + methodName = "getProcessCpuTimeByNS"; + } + } + final Method method = mbeanClazz + .getMethod(methodName); + return method; + } catch (ClassNotFoundException e) { + return null; + } catch (SecurityException e) { + return null; + } catch (NoSuchMethodException e) { + return null; + } + } }
