Repository: tez Updated Branches: refs/heads/branch-0.7 e2cf5cc70 -> c65a9c5e4
TEZ-2767. Make TezMxBeanResourceCalculator the default resource calculator. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c65a9c5e Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c65a9c5e Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c65a9c5e Branch: refs/heads/branch-0.7 Commit: c65a9c5e4eab37dd6360c299e5c980fded335fe2 Parents: e2cf5cc Author: Siddharth Seth <[email protected]> Authored: Wed Sep 2 17:28:11 2015 -0700 Committer: Siddharth Seth <[email protected]> Committed: Wed Sep 2 17:28:11 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + tez-common/pom.xml | 10 ++- .../tez/util/TezMxBeanResourceCalculator.java | 79 ++++++++++++++++++++ .../util/TestTezMxBeanResourceCalculator.java | 61 +++++++++++++++ .../org/apache/tez/dag/app/DAGAppMaster.java | 4 +- tez-plugins/pom.xml | 24 ------ .../findbugs-exclude.xml | 16 ---- .../tez-mbeans-resource-calculator/pom.xml | 60 --------------- .../tez/util/TezMxBeanResourceCalculator.java | 79 -------------------- .../util/TestTezMxBeanResourceCalculator.java | 61 --------------- .../tez/runtime/metrics/TaskCounterUpdater.java | 4 +- 11 files changed, 155 insertions(+), 244 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 9c30d54..0c68990 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2767. Make TezMxBeanResourceCalculator the default resource calculator. TEZ-2755. Fix findbugs warning in TezClient TEZ-2602. Throwing EOFException when launching MR job TEZ-2575. Handle KeyValue pairs size which do not fit in a single block in PipelinedSorter http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-common/pom.xml ---------------------------------------------------------------------- diff --git a/tez-common/pom.xml b/tez-common/pom.xml index 115c1a0..40bd3e2 100644 --- a/tez-common/pom.xml +++ b/tez-common/pom.xml @@ -54,8 +54,14 @@ <artifactId>tez-api</artifactId> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java new file mode 100644 index 0000000..1f8e922 --- /dev/null +++ b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.util; + +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; + +import java.lang.management.ManagementFactory; +import java.util.concurrent.TimeUnit; + +/** + * Uses sun's MBeans to return process information. + */ +public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { + + private final com.sun.management.OperatingSystemMXBean osBean; + private final Runtime runtime; + + /** + * Create process-tree instance with specified root process. + * <p/> + * Subclass must override this. + * + * @param root process-tree root-process + */ + public TezMxBeanResourceCalculator(String root) { + super(root); + runtime = Runtime.getRuntime(); + osBean = + (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + } + + @Override public void updateProcessTree() { + //nothing needs to be done as the data is read from OS mbeans. + } + + @Override public String getProcessTreeDump() { + return ""; + } + + @Override public long getCumulativeVmem(int olderThanAge) { + return osBean.getCommittedVirtualMemorySize(); + } + + @Override public long getCumulativeRssmem(int olderThanAge) { + //Not supported directly (RSS ~= memory consumed by JVM from Xmx) + return runtime.totalMemory(); + } + + @Override public long getCumulativeCpuTime() { + //convert to milliseconds + return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS); + } + + @Override public boolean checkPidPgrpidForMatch() { + return true; + } + + public float getCpuUsagePercent() { + //osBean.getProcessCpuLoad() can be closer and returns [0 - 1.0], but might not be accurate. + //Returning -1 to indicate, this feature is not yet supported. + return -1; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java new file mode 100644 index 0000000..4be8229 --- /dev/null +++ b/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.tez.dag.api.TezConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestTezMxBeanResourceCalculator { + + private ResourceCalculatorProcessTree resourceCalculator; + + @Before + public void setup() throws Exception { + Configuration conf = new TezConfiguration(); + conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, + TezMxBeanResourceCalculator.class.getName()); + + Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass( + TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, + ResourceCalculatorProcessTree.class); + resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree( + "", clazz, conf); + } + + @After + public void teardown() { + resourceCalculator = null; + } + + @Test(timeout=5000) + public void testResourceCalculator() { + Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator); + Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0); + Assert.assertTrue(resourceCalculator.getCumulativeVmem() > 0); + Assert.assertTrue(resourceCalculator.getCumulativeRssmem() > 0); + Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals("")); + Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch()); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 9bf0819..0e5c459 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -165,6 +165,7 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.dag.utils.Graph; import org.apache.tez.dag.utils.RelocalizationUtils; import org.apache.tez.dag.utils.Simple2LevelVersionComparator; +import org.apache.tez.util.TezMxBeanResourceCalculator; import org.codehaus.jettison.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -337,7 +338,8 @@ public class DAGAppMaster extends AbstractService { private void initResourceCalculatorPlugins() { Class<? extends ResourceCalculatorProcessTree> clazz = amConf.getClass( - TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, + TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, + TezMxBeanResourceCalculator.class, ResourceCalculatorProcessTree.class); // this is set by YARN NM http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml index 3bcaf37..89b9d1e 100644 --- a/tez-plugins/pom.xml +++ b/tez-plugins/pom.xml @@ -48,30 +48,6 @@ <module>tez-yarn-timeline-history-with-acls</module> </modules> </profile> - <profile> - <id>jdk6check</id> - <activation> - <property> - <name>java.vendor.url</name> - <value>http://java.sun.com/</value> - </property> - </activation> - <modules> - <module>tez-mbeans-resource-calculator</module> - </modules> - </profile> - <profile> - <id>jdk7check</id> - <activation> - <property> - <name>java.vendor.url</name> - <value>http://java.oracle.com/</value> - </property> - </activation> - <modules> - <module>tez-mbeans-resource-calculator</module> - </modules> - </profile> </profiles> <build> http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml b/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml deleted file mode 100644 index 5b11308..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<FindBugsFilter> - -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-plugins/tez-mbeans-resource-calculator/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/pom.xml b/tez-plugins/tez-mbeans-resource-calculator/pom.xml deleted file mode 100644 index e8fb4ac..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.tez</groupId> - <artifactId>tez-plugins</artifactId> - <version>0.7.1-SNAPSHOT</version> - </parent> - <artifactId>tez-mbeans-resource-calculator</artifactId> - - <dependencies> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>tez-dag</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java deleted file mode 100644 index 1f8e922..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.util; - -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; - -import java.lang.management.ManagementFactory; -import java.util.concurrent.TimeUnit; - -/** - * Uses sun's MBeans to return process information. - */ -public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { - - private final com.sun.management.OperatingSystemMXBean osBean; - private final Runtime runtime; - - /** - * Create process-tree instance with specified root process. - * <p/> - * Subclass must override this. - * - * @param root process-tree root-process - */ - public TezMxBeanResourceCalculator(String root) { - super(root); - runtime = Runtime.getRuntime(); - osBean = - (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); - } - - @Override public void updateProcessTree() { - //nothing needs to be done as the data is read from OS mbeans. - } - - @Override public String getProcessTreeDump() { - return ""; - } - - @Override public long getCumulativeVmem(int olderThanAge) { - return osBean.getCommittedVirtualMemorySize(); - } - - @Override public long getCumulativeRssmem(int olderThanAge) { - //Not supported directly (RSS ~= memory consumed by JVM from Xmx) - return runtime.totalMemory(); - } - - @Override public long getCumulativeCpuTime() { - //convert to milliseconds - return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS); - } - - @Override public boolean checkPidPgrpidForMatch() { - return true; - } - - public float getCpuUsagePercent() { - //osBean.getProcessCpuLoad() can be closer and returns [0 - 1.0], but might not be accurate. - //Returning -1 to indicate, this feature is not yet supported. - return -1; - } -} http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java deleted file mode 100644 index 4be8229..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.util; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; -import org.apache.tez.dag.api.TezConfiguration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestTezMxBeanResourceCalculator { - - private ResourceCalculatorProcessTree resourceCalculator; - - @Before - public void setup() throws Exception { - Configuration conf = new TezConfiguration(); - conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, - TezMxBeanResourceCalculator.class.getName()); - - Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass( - TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, - ResourceCalculatorProcessTree.class); - resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree( - "", clazz, conf); - } - - @After - public void teardown() { - resourceCalculator = null; - } - - @Test(timeout=5000) - public void testResourceCalculator() { - Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator); - Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0); - Assert.assertTrue(resourceCalculator.getCumulativeVmem() > 0); - Assert.assertTrue(resourceCalculator.getCumulativeRssmem() > 0); - Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals("")); - Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch()); - } - -} http://git-wip-us.apache.org/repos/asf/tez/blob/c65a9c5e/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java index 6399ded..ebb94c6 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.tez.util.TezMxBeanResourceCalculator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -142,7 +143,8 @@ public class TaskCounterUpdater { private void initResourceCalculatorPlugin() { Class<? extends ResourceCalculatorProcessTree> clazz = this.conf.getClass( - TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, + TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, + TezMxBeanResourceCalculator.class, ResourceCalculatorProcessTree.class); pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, conf);
