Repository: tez Updated Branches: refs/heads/master 202393418 -> 58aaa36f2
TEZ-2767. Make TezMxBeanResourceCalculator the default resource calculator. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/58aaa36f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/58aaa36f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/58aaa36f Branch: refs/heads/master Commit: 58aaa36f28149ed0285e6fb13fea4846c7024c25 Parents: 2023934 Author: Siddharth Seth <[email protected]> Authored: Wed Sep 2 11:40:04 2015 -0700 Committer: Siddharth Seth <[email protected]> Committed: Wed Sep 2 11:40:04 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 + tez-common/pom.xml | 10 ++- .../tez/util/TezMxBeanResourceCalculator.java | 79 ++++++++++++++++++++ .../util/TestTezMxBeanResourceCalculator.java | 61 +++++++++++++++ .../org/apache/tez/dag/app/DAGAppMaster.java | 4 +- tez-plugins/pom.xml | 24 ------ .../findbugs-exclude.xml | 16 ---- .../tez-mbeans-resource-calculator/pom.xml | 60 --------------- .../tez/util/TezMxBeanResourceCalculator.java | 79 -------------------- .../util/TestTezMxBeanResourceCalculator.java | 61 --------------- 10 files changed, 153 insertions(+), 243 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5f0ca2b..1f785b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ Release 0.8.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2767. Make TezMxBeanResourceCalculator the default resource calculator. TEZ-2765. Change Xmlwriter to use defaultValue instead of value tag. TEZ-2750. Shuffle may not shutdown in case of a fetch failure, causing it to hang. TEZ-2294. Add tez-site-template.xml with description of config properties. @@ -158,6 +159,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2767. Make TezMxBeanResourceCalculator the default resource calculator. TEZ-2602. Throwing EOFException when launching MR job TEZ-2575. Handle KeyValue pairs size which do not fit in a single block in PipelinedSorter TEZ-2198. Fix sorter spill counts http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-common/pom.xml ---------------------------------------------------------------------- diff --git a/tez-common/pom.xml b/tez-common/pom.xml index fceb589..49846ca 100644 --- a/tez-common/pom.xml +++ b/tez-common/pom.xml @@ -54,8 +54,14 @@ <artifactId>tez-api</artifactId> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java new file mode 100644 index 0000000..1f8e922 --- /dev/null +++ b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.util; + +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; + +import java.lang.management.ManagementFactory; +import java.util.concurrent.TimeUnit; + +/** + * Uses sun's MBeans to return process information. + */ +public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { + + private final com.sun.management.OperatingSystemMXBean osBean; + private final Runtime runtime; + + /** + * Create process-tree instance with specified root process. + * <p/> + * Subclass must override this. + * + * @param root process-tree root-process + */ + public TezMxBeanResourceCalculator(String root) { + super(root); + runtime = Runtime.getRuntime(); + osBean = + (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + } + + @Override public void updateProcessTree() { + //nothing needs to be done as the data is read from OS mbeans. + } + + @Override public String getProcessTreeDump() { + return ""; + } + + @Override public long getCumulativeVmem(int olderThanAge) { + return osBean.getCommittedVirtualMemorySize(); + } + + @Override public long getCumulativeRssmem(int olderThanAge) { + //Not supported directly (RSS ~= memory consumed by JVM from Xmx) + return runtime.totalMemory(); + } + + @Override public long getCumulativeCpuTime() { + //convert to milliseconds + return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS); + } + + @Override public boolean checkPidPgrpidForMatch() { + return true; + } + + public float getCpuUsagePercent() { + //osBean.getProcessCpuLoad() can be closer and returns [0 - 1.0], but might not be accurate. + //Returning -1 to indicate, this feature is not yet supported. + return -1; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java new file mode 100644 index 0000000..4be8229 --- /dev/null +++ b/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.tez.dag.api.TezConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestTezMxBeanResourceCalculator { + + private ResourceCalculatorProcessTree resourceCalculator; + + @Before + public void setup() throws Exception { + Configuration conf = new TezConfiguration(); + conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, + TezMxBeanResourceCalculator.class.getName()); + + Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass( + TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, + ResourceCalculatorProcessTree.class); + resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree( + "", clazz, conf); + } + + @After + public void teardown() { + resourceCalculator = null; + } + + @Test(timeout=5000) + public void testResourceCalculator() { + Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator); + Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0); + Assert.assertTrue(resourceCalculator.getCumulativeVmem() > 0); + Assert.assertTrue(resourceCalculator.getCumulativeRssmem() > 0); + Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals("")); + Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch()); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 613046b..04c7b82 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -172,6 +172,7 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.dag.utils.Graph; import org.apache.tez.dag.utils.RelocalizationUtils; import org.apache.tez.dag.utils.Simple2LevelVersionComparator; +import org.apache.tez.util.TezMxBeanResourceCalculator; import org.codehaus.jettison.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -351,7 +352,8 @@ public class DAGAppMaster extends AbstractService { private void initResourceCalculatorPlugins() { Class<? extends ResourceCalculatorProcessTree> clazz = amConf.getClass( - TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, + TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, + TezMxBeanResourceCalculator.class, ResourceCalculatorProcessTree.class); // this is set by YARN NM http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml index 657b8bf..7114bdc 100644 --- a/tez-plugins/pom.xml +++ b/tez-plugins/pom.xml @@ -50,30 +50,6 @@ <module>tez-history-parser</module> </modules> </profile> - <profile> - <id>jdk6check</id> - <activation> - <property> - <name>java.vendor.url</name> - <value>http://java.sun.com/</value> - </property> - </activation> - <modules> - <module>tez-mbeans-resource-calculator</module> - </modules> - </profile> - <profile> - <id>jdk7check</id> - <activation> - <property> - <name>java.vendor.url</name> - <value>http://java.oracle.com/</value> - </property> - </activation> - <modules> - <module>tez-mbeans-resource-calculator</module> - </modules> - </profile> </profiles> <build> http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml b/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml deleted file mode 100644 index 5b11308..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<FindBugsFilter> - -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-plugins/tez-mbeans-resource-calculator/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/pom.xml b/tez-plugins/tez-mbeans-resource-calculator/pom.xml deleted file mode 100644 index 6783803..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.tez</groupId> - <artifactId>tez-plugins</artifactId> - <version>0.8.1-SNAPSHOT</version> - </parent> - <artifactId>tez-mbeans-resource-calculator</artifactId> - - <dependencies> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>tez-dag</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java deleted file mode 100644 index 1f8e922..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.util; - -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; - -import java.lang.management.ManagementFactory; -import java.util.concurrent.TimeUnit; - -/** - * Uses sun's MBeans to return process information. - */ -public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree { - - private final com.sun.management.OperatingSystemMXBean osBean; - private final Runtime runtime; - - /** - * Create process-tree instance with specified root process. - * <p/> - * Subclass must override this. - * - * @param root process-tree root-process - */ - public TezMxBeanResourceCalculator(String root) { - super(root); - runtime = Runtime.getRuntime(); - osBean = - (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); - } - - @Override public void updateProcessTree() { - //nothing needs to be done as the data is read from OS mbeans. - } - - @Override public String getProcessTreeDump() { - return ""; - } - - @Override public long getCumulativeVmem(int olderThanAge) { - return osBean.getCommittedVirtualMemorySize(); - } - - @Override public long getCumulativeRssmem(int olderThanAge) { - //Not supported directly (RSS ~= memory consumed by JVM from Xmx) - return runtime.totalMemory(); - } - - @Override public long getCumulativeCpuTime() { - //convert to milliseconds - return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS); - } - - @Override public boolean checkPidPgrpidForMatch() { - return true; - } - - public float getCpuUsagePercent() { - //osBean.getProcessCpuLoad() can be closer and returns [0 - 1.0], but might not be accurate. - //Returning -1 to indicate, this feature is not yet supported. - return -1; - } -} http://git-wip-us.apache.org/repos/asf/tez/blob/58aaa36f/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java deleted file mode 100644 index 4be8229..0000000 --- a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.util; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; -import org.apache.tez.dag.api.TezConfiguration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestTezMxBeanResourceCalculator { - - private ResourceCalculatorProcessTree resourceCalculator; - - @Before - public void setup() throws Exception { - Configuration conf = new TezConfiguration(); - conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, - TezMxBeanResourceCalculator.class.getName()); - - Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass( - TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, - ResourceCalculatorProcessTree.class); - resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree( - "", clazz, conf); - } - - @After - public void teardown() { - resourceCalculator = null; - } - - @Test(timeout=5000) - public void testResourceCalculator() { - Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator); - Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0); - Assert.assertTrue(resourceCalculator.getCumulativeVmem() > 0); - Assert.assertTrue(resourceCalculator.getCumulativeRssmem() > 0); - Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals("")); - Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch()); - } - -}
