Repository: tez Updated Branches: refs/heads/master 7b16b77aa -> 7940facf6
TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7940facf Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7940facf Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7940facf Branch: refs/heads/master Commit: 7940facf676087eab0301d6ada96d0b6576ca693 Parents: 7b16b77 Author: Hitesh Shah <[email protected]> Authored: Fri Dec 11 16:34:16 2015 -0800 Committer: Hitesh Shah <[email protected]> Committed: Fri Dec 11 16:34:55 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + hadoop-shim-impls/hadoop-shim-2.4/pom.xml | 6 ++ .../shim/TestHadoop23_24ShimProvider.java | 44 ++++++++++++++ hadoop-shim-impls/hadoop-shim-2.6/pom.xml | 6 ++ .../shim/TestHadoop25_26_27ShimProvider.java | 42 +++++++++++++ hadoop-shim-impls/hadoop-shim-2.8/pom.xml | 6 ++ .../hadoop/shim/TestHadoopShim28Provider.java | 42 +++++++++++++ .../tez/hadoop/shim/HadoopShimsLoader.java | 63 +++++++++++++++++++- .../tez/hadoop/shim/DummyShimProvider.java | 30 ++++++++++ .../tez/hadoop/shim/TestHadoopShimsLoader.java | 54 +++++++++++++++++ hadoop-shim/src/test/resources/log4j.properties | 19 ++++++ pom.xml | 6 ++ tez-dag/findbugs-exclude.xml | 1 + .../org/apache/tez/dag/app/DAGAppMaster.java | 6 +- 14 files changed, 322 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b19c4be..1e1537f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,6 +10,7 @@ INCOMPATIBLE CHANGES TEZ-604. Revert temporary changes made in TEZ-603 to kill the provided tez session, if running a MapReduce job. ALL CHANGES: + TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations. TEZ-2472. Change slf4j version to 1.7.10. TEZ-2920. org.apache.tez.client.TestTezClient.testStopRetriesUntilTimeout is flaky. TEZ-2824. Add javadocs for Vertex.setConf and DAG.setConf. http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.4/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml index 095c2b1..3ffe4e2 100644 --- a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml +++ b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml @@ -38,6 +38,12 @@ <artifactId>hadoop-shim</artifactId> </dependency> <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java index 9bc4435..871fb67 100644 --- a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java +++ b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java @@ -18,6 +18,8 @@ package org.apache.tez.hadoop.shim; +import org.apache.hadoop.conf.Configuration; +import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim; import org.junit.Assert; import org.junit.Test; @@ -33,6 +35,48 @@ public class TestHadoop23_24ShimProvider { Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3)); Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4)); + + Assert.assertEquals(HadoopShim24.class, + provider.createHadoopShim("foo", 2, 3).getClass()); + + } + + @Test + public void testLoaderOverride() { + Configuration conf = new Configuration(false); + // Set shim and version to ensure that hadoop version from the build does not create + // a mismatch + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim23_24Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.4.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(HadoopShim24.class, shim.getClass()); + } + + @Test + public void testInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that shim in this module does not match + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.6.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); + } + + @Test + public void testLoaderOverrideInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that override shim does not return a valid shim + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim23_24Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); } } http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.6/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml index 17f4711..8346c37 100644 --- a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml +++ b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml @@ -38,6 +38,12 @@ <artifactId>hadoop-shim</artifactId> </dependency> <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java index 0e6d997..13faf37 100644 --- a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java +++ b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java @@ -18,6 +18,7 @@ package org.apache.tez.hadoop.shim; +import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; @@ -34,6 +35,47 @@ public class TestHadoop25_26_27ShimProvider { Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5)); Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6)); Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7)); + + Assert.assertEquals(HadoopShim26.class, + provider.createHadoopShim("foo", 2, 7).getClass()); + } + + @Test + public void testLoaderOverride() { + Configuration conf = new Configuration(false); + // Set shim and version to ensure that hadoop version from the build does not create + // a mismatch + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim25_26_27Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(HadoopShim26.class, shim.getClass()); + } + + @Test + public void testInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that shim in this module does not match + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); + } + + @Test + public void testLoaderOverrideInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that override shim does not return a valid shim + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim25_26_27Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); } } http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.8/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml index 18d22c5..41928fe 100644 --- a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml +++ b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml @@ -38,6 +38,12 @@ <artifactId>hadoop-shim</artifactId> </dependency> <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java index df92627..353d590 100644 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java @@ -18,6 +18,7 @@ package org.apache.tez.hadoop.shim; +import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; @@ -33,6 +34,47 @@ public class TestHadoopShim28Provider { Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8)); Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111)); + + Assert.assertEquals(HadoopShim28.class, + provider.createHadoopShim("foo", 2, 9).getClass()); + } + + @Test + public void testLoaderOverride() { + Configuration conf = new Configuration(false); + // Set shim and version to ensure that hadoop version from the build does not create + // a mismatch + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim28Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.8.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(HadoopShim28.class, shim.getClass()); + } + + @Test + public void testInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that shim in this module does not match + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.2.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); + } + + @Test + public void testLoaderOverrideInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that override shim does not return a valid shim + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim28Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); } } http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java index fc2ec81..9d9dc3f 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java @@ -22,10 +22,13 @@ import java.util.ServiceLoader; import java.util.StringTokenizer; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.VersionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + @Private public class HadoopShimsLoader { @@ -37,17 +40,57 @@ public class HadoopShimsLoader { private final HadoopShim currentShim; private final HadoopShimProvider currentShimProvider; - public HadoopShimsLoader() { + public static final String TEZ_HADOOP_SHIM_PROVIDER_CLASS = + "tez.hadoop.shim.provider.class"; + + // A way to override the hadoop version for testing + @Private + @VisibleForTesting + static final String TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE = + "tez.hadoop.shim.hadoop.version.override"; + + public HadoopShimsLoader(Configuration conf) { + this(conf, false); + } + + @VisibleForTesting + @SuppressWarnings("unchecked") + HadoopShimsLoader(Configuration conf, boolean useReflection) { + String overrideProviderClassStr = conf.get(TEZ_HADOOP_SHIM_PROVIDER_CLASS); + if (overrideProviderClassStr != null && !overrideProviderClassStr.isEmpty()) { + LOG.info("HadoopShim Selection is overridden, using Configured Provider=" + + overrideProviderClassStr); + } else { + overrideProviderClassStr = null; + } + String versionStr = VersionInfo.getVersion(); + String overrideVersionStr = conf.get(TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE); + if (overrideVersionStr != null && !overrideVersionStr.isEmpty()) { + LOG.warn("Using overridden hadoop version instead of actual version" + + ", realVersion=" + versionStr + + ", overrideVersion=" + overrideVersionStr); + versionStr = overrideVersionStr; + } + Version version = new Version(versionStr); HadoopShim selectedShim = null; HadoopShimProvider selectedShimProvider = null; + LOG.info("Trying to locate HadoopShimProvider for " + "hadoopVersion=" + versionStr + ", majorVersion=" + version.majorVersion + ", minorVersion=" + version.minorVersion); synchronized (shimLoader) { for (HadoopShimProvider provider : shimLoader) { + if (overrideProviderClassStr != null + && !provider.getClass().getName().equals(overrideProviderClassStr)) { + LOG.debug("Skipping HadoopShimProvider : " + + provider.getClass().getName() + + " as config provided to override selection"); + continue; + } + LOG.debug("Trying HadoopShimProvider : " + provider.getClass().getName()); HadoopShim hadoopShim = null; @@ -70,6 +113,21 @@ public class HadoopShimsLoader { + " due to error: ", e); } } + if (selectedShim == null && useReflection && overrideProviderClassStr != null) { + try { + LOG.debug("Using Reflection to create HadoopShim from provider class=" + + overrideProviderClassStr); + Class<HadoopShimProvider> clazz = (Class<HadoopShimProvider>)Class.forName( + overrideProviderClassStr, true, Thread.currentThread().getContextClassLoader()); + selectedShimProvider = clazz.newInstance(); + selectedShim = selectedShimProvider.createHadoopShim(versionStr, + version.majorVersion, version.minorVersion); + } catch (Exception e) { + throw new RuntimeException("Unable to create HadoopShim from provider class: " + + overrideProviderClassStr, e); + } + } + if (selectedShim == null) { currentShim = new DefaultHadoopShim(); currentShimProvider = null; @@ -80,7 +138,8 @@ public class HadoopShimsLoader { } LOG.info("Picked HadoopShim " + currentShim.getClass().getName() + ", providerName=" - + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null" ) + + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null") + + ", overrideProviderViaConfig=" + overrideProviderClassStr + ", hadoopVersion=" + versionStr + ", majorVersion=" + version.majorVersion + ", minorVersion=" + version.minorVersion); http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java new file mode 100644 index 0000000..7b72c7a --- /dev/null +++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +public class DummyShimProvider extends HadoopShimProvider { + + public static class DummyShim extends HadoopShim { + } + + @Override + public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { + return new DummyShim(); + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java new file mode 100644 index 0000000..b0215ed --- /dev/null +++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.conf.Configuration; +import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim; +import org.junit.Assert; +import org.junit.Test; + +public class TestHadoopShimsLoader { + + @Test + public void testBasicLoader() { + HadoopShimsLoader loader = new HadoopShimsLoader(new Configuration(false)); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); + } + + @Test + public void testLoaderOverride() { + Configuration conf = new Configuration(false); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, DummyShimProvider.class.getName()); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DummyShim.class, shim.getClass()); + } + + @Test(expected = RuntimeException.class) + public void testInvalidOverride() { + Configuration conf = new Configuration(false); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, "org.apache.tez.foo"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/test/resources/log4j.properties b/hadoop-shim/src/test/resources/log4j.properties new file mode 100644 index 0000000..63353bd --- /dev/null +++ b/hadoop-shim/src/test/resources/log4j.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=debug,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ac449e2..7759de5 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,12 @@ </dependency> <dependency> <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> <artifactId>tez-api</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/tez-dag/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml index 2122c5e..43046d9 100644 --- a/tez-dag/findbugs-exclude.xml +++ b/tez-dag/findbugs-exclude.xml @@ -161,6 +161,7 @@ <Field name="nodes"/> <Field name="recoveryEnabled"/> <Field name="isLocal"/> + <Field name="hadoopShim"/> </Or> <Bug pattern="IS2_INCONSISTENT_SYNC"/> </Match> http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 05261ba..5cbdab4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -240,7 +240,7 @@ public class DAGAppMaster extends AbstractService { private final String[] localDirs; private final String[] logDirs; private final AMPluginDescriptorProto amPluginDescriptorProto; - private final HadoopShim hadoopShim; + private HadoopShim hadoopShim; private ContainerSignatureMatcher containerSignatureMatcher; private AMContainerMap containers; private AMNodeTracker nodes; @@ -360,7 +360,6 @@ public class DAGAppMaster extends AbstractService { this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort, this.containerID.toString(), this.appMasterUgi.getShortUserName()); - this.hadoopShim = new HadoopShimsLoader().getHadoopShim(); LOG.info("Created DAGAppMaster for application " + applicationAttemptId + ", versionInfo=" + dagVersionInfo.toString()); @@ -417,6 +416,9 @@ public class DAGAppMaster extends AbstractService { this.amConf = conf; initResourceCalculatorPlugins(); + this.hadoopShim = new HadoopShimsLoader(this.amConf).getHadoopShim(); + + this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
