Repository: tez Updated Branches: refs/heads/master c9c6fbf5f -> ca607d65e
TEZ-3689. Change minimum hadoop version to 2.7.0. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ca607d65 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ca607d65 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ca607d65 Branch: refs/heads/master Commit: ca607d65ebfb4e0309a33b59ad9fcc004222b4a5 Parents: c9c6fbf Author: Siddharth Seth <[email protected]> Authored: Thu Apr 20 16:31:57 2017 -0700 Committer: Siddharth Seth <[email protected]> Committed: Thu Apr 20 16:31:57 2017 -0700 ---------------------------------------------------------------------- BUILDING.txt | 12 +-- docs/src/site/markdown/install.md | 4 +- .../hadoop-shim-2.4/findbugs-exclude.xml | 16 ---- hadoop-shim-impls/hadoop-shim-2.4/pom.xml | 56 ------------- .../hadoop/shim/HadoopShim23_24Provider.java | 33 -------- .../apache/tez/hadoop/shim/HadoopShim24.java | 45 ----------- ...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 ---- .../shim/TestHadoop23_24ShimProvider.java | 82 -------------------- .../hadoop-shim-2.6/findbugs-exclude.xml | 16 ---- hadoop-shim-impls/hadoop-shim-2.6/pom.xml | 56 ------------- .../hadoop/shim/HadoopShim25_26_27Provider.java | 33 -------- .../apache/tez/hadoop/shim/HadoopShim26.java | 52 ------------- ...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 ---- .../shim/TestHadoop25_26_27ShimProvider.java | 81 ------------------- .../hadoop-shim-2.7/findbugs-exclude.xml | 16 ++++ hadoop-shim-impls/hadoop-shim-2.7/pom.xml | 56 +++++++++++++ .../hadoop/shim/HadoopShim25_26_27Provider.java | 33 ++++++++ .../apache/tez/hadoop/shim/HadoopShim27.java | 52 +++++++++++++ ...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 ++++ .../shim/TestHadoop25_26_27ShimProvider.java | 81 +++++++++++++++++++ hadoop-shim-impls/pom.xml | 13 +--- pom.xml | 2 +- .../api/client/TestTimelineReaderFactory.java | 29 ------- tez-dist/pom.xml | 22 +----- tez-plugins/pom.xml | 12 +-- tez-plugins/tez-yarn-timeline-history/pom.xml | 2 +- tez-tools/analyzers/pom.xml | 11 +-- 27 files changed, 269 insertions(+), 588 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt index 0907b6e..34ed2fb 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -11,7 +11,7 @@ Requirements: * Findbugs 2.0.2 or later (if running findbugs) * ProtocolBuffer 2.5.0 * Internet connection for first build (to fetch all dependencies) -* Hadoop version should be 2.6.0 or higher. +* Hadoop version should be 2.7.0 or higher. ---------------------------------------------------------------------------------- Maven main modules: @@ -43,7 +43,7 @@ Maven build goals: * Run clover : mvn test -Pclover [-Dclover.license=${user.home}/clover.license] * Run Rat : mvn apache-rat:check * Build javadocs : mvn javadoc:javadoc - * Build distribution : mvn package[-Dhadoop.version=2.6.0] + * Build distribution : mvn package[-Dhadoop.version=2.7.0] * Visualize state machines : mvn compile -Pvisualize -DskipTests=true Build options: @@ -64,7 +64,7 @@ Tests options: ---------------------------------------------------------------------------------- Building against a specific version of hadoop: -Tez runs on top of Apache Hadoop YARN and requires hadoop version 2.6.0 or higher. +Tez runs on top of Apache Hadoop YARN and requires hadoop version 2.7.0 or higher. By default, it can be compiled against other compatible hadoop versions by just specifying the hadoop.version. For example, to build tez against hadoop 3.0.0-SNAPSHOT @@ -75,15 +75,15 @@ To skip Tests and java docs $ mvn package -Dhadoop.version=3.0.0-SNAPSHOT -DskipTests -Dmaven.javadoc.skip=true -However, to build against hadoop versions lower or higher than 2.6.0, you will need to do the +However, to build against hadoop versions higher than 2.7.0, you will need to do the following: For Hadoop version X where X >= 2.8.0 - $ mvn package -Dhadoop.version=${X} -Phadoop28 -P\!hadoop26 + $ mvn package -Dhadoop.version=${X} -Phadoop28 -P\!hadoop27 For recent versions of Hadoop (which do not bundle aws and azure by default), -you can bundle AWS-S3 (2.6.0+) or Azure (2.7.0+) support: +you can bundle AWS-S3 (2.7.0+) or Azure (2.7.0+) support: $ mvn package -Dhadoop.version=${X} -Paws -Pazure http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/docs/src/site/markdown/install.md ---------------------------------------------------------------------- diff --git a/docs/src/site/markdown/install.md b/docs/src/site/markdown/install.md index 4eb52ec..aadfa29 100644 --- a/docs/src/site/markdown/install.md +++ b/docs/src/site/markdown/install.md @@ -21,8 +21,10 @@ Install/Deploy Instructions for Tez --------------------------------------------------------------------------- Replace x.y.z with the tez release number that you are using. E.g. 0.5.0. For Tez versions 0.8.3 and higher, Tez needs Apache Hadoop to be of version 2.6.0 or higher. +For Tez version 0.9.0 and higher, Tez needs Apache Hadoop to be version 2.7.0 +or higher. -1. Deploy Apache Hadoop using version of 2.6.0 or higher. +1. Deploy Apache Hadoop using version of 2.7.0 or higher. - You need to change the value of the hadoop.version property in the top-level pom.xml to match the version of the hadoop branch being used. http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml deleted file mode 100644 index 5b11308..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<FindBugsFilter> - -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.4/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml deleted file mode 100644 index 8b42f78..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>hadoop-shim-impls</artifactId> - <groupId>org.apache.tez</groupId> - <version>0.9.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>hadoop-shim-2.4</artifactId> - - <dependencies> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>hadoop-shim</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>hadoop-shim</artifactId> - <scope>test</scope> - <type>test-jar</type> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - </dependencies> - -</project> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java deleted file mode 100644 index d94b5bf..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.classification.InterfaceAudience.Private; - -@Private -public class HadoopShim23_24Provider extends HadoopShimProvider { - - @Override - public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { - if (majorVersion == 2 && (minorVersion == 3 || minorVersion == 4)) { - return new org.apache.tez.hadoop.shim.HadoopShim24(); - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java deleted file mode 100644 index 3f7e56e..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; - -@Private -public class HadoopShim24 extends HadoopShim { - - @Override - public void setHadoopCallerContext(String context) { - // Not supported - } - - @Override - public void clearHadoopCallerContext() { - // Not supported - } - - @Override - public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { - // Not supported - return null; - } - -} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider deleted file mode 100644 index a675e20..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.tez.hadoop.shim.HadoopShim23_24Provider http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java deleted file mode 100644 index 871fb67..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.conf.Configuration; -import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim; -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoop23_24ShimProvider { - - @Test - public void testShimProvider() { - HadoopShim23_24Provider provider = new HadoopShim23_24Provider(); - Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 5)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 6)); - Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4)); - - Assert.assertEquals(HadoopShim24.class, - provider.createHadoopShim("foo", 2, 3).getClass()); - - } - - @Test - public void testLoaderOverride() { - Configuration conf = new Configuration(false); - // Set shim and version to ensure that hadoop version from the build does not create - // a mismatch - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim23_24Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.4.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(HadoopShim24.class, shim.getClass()); - } - - @Test - public void testInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that shim in this module does not match - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.6.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - - @Test - public void testLoaderOverrideInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that override shim does not return a valid shim - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim23_24Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - -} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml deleted file mode 100644 index 5b11308..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<FindBugsFilter> - -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.6/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml deleted file mode 100644 index 864165b..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>hadoop-shim-impls</artifactId> - <groupId>org.apache.tez</groupId> - <version>0.9.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>hadoop-shim-2.6</artifactId> - - <dependencies> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>hadoop-shim</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>hadoop-shim</artifactId> - <scope>test</scope> - <type>test-jar</type> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - </dependencies> - -</project> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java deleted file mode 100644 index ab013e9..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.classification.InterfaceAudience.Private; - -@Private -public class HadoopShim25_26_27Provider extends HadoopShimProvider { - - @Override - public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { - if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) { - return new HadoopShim26(); - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java deleted file mode 100644 index c1622dc..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; - -@Private -public class HadoopShim26 extends HadoopShim { - - @Override - public void setHadoopCallerContext(String context) { - // Not supported - } - - @Override - public void clearHadoopCallerContext() { - // Not supported - } - - @Override - public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { - EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes(); - Set<String> supportedTypes = new HashSet<String>(); - for (SchedulerResourceTypes resourceType : supportedResourceTypes) { - supportedTypes.add(resourceType.name()); - } - return supportedTypes; - } - -} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider deleted file mode 100644 index 4226f18..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java deleted file mode 100644 index 13faf37..0000000 --- a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoop25_26_27ShimProvider { - - @Test - public void testShimProvider() { - HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider(); - Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 3)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 8)); - Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7)); - - Assert.assertEquals(HadoopShim26.class, - provider.createHadoopShim("foo", 2, 7).getClass()); - } - - @Test - public void testLoaderOverride() { - Configuration conf = new Configuration(false); - // Set shim and version to ensure that hadoop version from the build does not create - // a mismatch - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim25_26_27Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(HadoopShim26.class, shim.getClass()); - } - - @Test - public void testInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that shim in this module does not match - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - - @Test - public void testLoaderOverrideInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that override shim does not return a valid shim - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim25_26_27Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - -} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.7/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.7/pom.xml b/hadoop-shim-impls/hadoop-shim-2.7/pom.xml new file mode 100644 index 0000000..7c6d9ab --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.7/pom.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hadoop-shim-impls</artifactId> + <groupId>org.apache.tez</groupId> + <version>0.9.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>hadoop-shim-2.7</artifactId> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java new file mode 100644 index 0000000..bd1eabf --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +@Private +public class HadoopShim25_26_27Provider extends HadoopShimProvider { + + @Override + public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { + if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) { + return new HadoopShim27(); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java new file mode 100644 index 0000000..c516303 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; + +@Private +public class HadoopShim27 extends HadoopShim { + + @Override + public void setHadoopCallerContext(String context) { + // Not supported + } + + @Override + public void clearHadoopCallerContext() { + // Not supported + } + + @Override + public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes(); + Set<String> supportedTypes = new HashSet<String>(); + for (SchedulerResourceTypes resourceType : supportedResourceTypes) { + supportedTypes.add(resourceType.name()); + } + return supportedTypes; + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider new file mode 100644 index 0000000..4226f18 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java new file mode 100644 index 0000000..c9937f8 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +public class TestHadoop25_26_27ShimProvider { + + @Test + public void testShimProvider() { + HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider(); + Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 3)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 8)); + Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7)); + + Assert.assertEquals(HadoopShim27.class, + provider.createHadoopShim("foo", 2, 7).getClass()); + } + + @Test + public void testLoaderOverride() { + Configuration conf = new Configuration(false); + // Set shim and version to ensure that hadoop version from the build does not create + // a mismatch + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim25_26_27Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(HadoopShim27.class, shim.getClass()); + } + + @Test + public void testInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that shim in this module does not match + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); + } + + @Test + public void testLoaderOverrideInvalidVersion() { + Configuration conf = new Configuration(false); + // Set incompatible version so that override shim does not return a valid shim + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, + HadoopShim25_26_27Provider.class.getName()); + conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); + HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); + HadoopShim shim = loader.getHadoopShim(); + Assert.assertNotNull(shim); + Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/hadoop-shim-impls/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/pom.xml b/hadoop-shim-impls/pom.xml index a2ab5ba..4baea3f 100644 --- a/hadoop-shim-impls/pom.xml +++ b/hadoop-shim-impls/pom.xml @@ -27,21 +27,12 @@ <profiles> <profile> - <id>hadoop24</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>hadoop-shim-2.4</module> - </modules> - </profile> - <profile> - <id>hadoop26</id> + <id>hadoop27</id> <activation> <activeByDefault>true</activeByDefault> </activation> <modules> - <module>hadoop-shim-2.6</module> + <module>hadoop-shim-2.7</module> </modules> </profile> <profile> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 22647cf..0307a04 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ <properties> <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile> <clover.license>${user.home}/clover.license</clover.license> - <hadoop.version>2.6.0</hadoop.version> + <hadoop.version>2.7.0</hadoop.version> <jetty.version>6.1.26</jetty.version> <pig.version>0.13.0</pig.version> <javac.version>1.7</javac.version> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java index 4aff0ca..b6d1396 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java @@ -29,43 +29,14 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.client.TimelineReaderFactory.TimelineReaderPseudoAuthenticatedStrategy; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; import org.junit.Test; public class TestTimelineReaderFactory { - @Before - public void setup() { - // Disable tests if hadoop version is less than 2.4.0 - // as Timeline is not supported in 2.2.x or 2.3.x - String hadoopVersion = System.getProperty("tez.hadoop.version"); - Assume.assumeFalse(hadoopVersion.startsWith("2.2.") || hadoopVersion.startsWith("2.3.")); - } - - // ensure on hadoop 2.4 TimelinePseudoAuthenticatedStrategy is used. - @Test(timeout = 5000) - public void testShouldUsePseudoAuthStrategyForHadoop24() throws TezException { - String hadoopVersion = System.getProperty("tez.hadoop.version"); - Assume.assumeTrue(hadoopVersion.startsWith("2.4.") || hadoopVersion.startsWith("2.5.")); - - String returnedClassName = - TimelineReaderFactory.getTimelineReaderStrategy(mock(Configuration.class), false, 0) - .getClass() - .getCanonicalName(); - Assert.assertEquals("should use pseudo auth on hadoop2.4", - "org.apache.tez.dag.api.client.TimelineReaderFactory.TimelineReaderPseudoAuthenticatedStrategy", - returnedClassName); - } - // ensure on hadoop 2.6+ TimelineReaderTokenAuthenticatedStrategy is used. @Test(timeout = 5000) public void testShouldUseTokenDelegationAuthStrategyForHadoop26() throws TezException { - String hadoopVersion = System.getProperty("tez.hadoop.version"); - Assume.assumeFalse(hadoopVersion.startsWith("2.2.") || - hadoopVersion.startsWith("2.3.") || - hadoopVersion.startsWith("2.4.") || - hadoopVersion.startsWith("2.5.")); String returnedClassName = TimelineReaderFactory.getTimelineReaderStrategy(mock(Configuration.class), false, 0) http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/tez-dist/pom.xml ---------------------------------------------------------------------- diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml index 926d152..f6c8b11 100644 --- a/tez-dist/pom.xml +++ b/tez-dist/pom.xml @@ -57,25 +57,7 @@ <profiles> <profile> - <id>hadoop24</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>tez-yarn-timeline-history</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>hadoop-shim-2.4</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>hadoop26</id> + <id>hadoop27</id> <activation> <activeByDefault>true</activeByDefault> </activation> @@ -87,7 +69,7 @@ </dependency> <dependency> <groupId>org.apache.tez</groupId> - <artifactId>hadoop-shim-2.6</artifactId> + <artifactId>hadoop-shim-2.7</artifactId> <version>${project.version}</version> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/tez-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml index 27707a8..eb2e242 100644 --- a/tez-plugins/pom.xml +++ b/tez-plugins/pom.xml @@ -28,17 +28,7 @@ <profiles> <profile> - <id>hadoop24</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>tez-yarn-timeline-history</module> - <module>tez-history-parser</module> - </modules> - </profile> - <profile> - <id>hadoop26</id> + <id>hadoop27</id> <activation> <property> <name>!skipATS</name> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/tez-plugins/tez-yarn-timeline-history/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml index e37883c..edfcddb 100644 --- a/tez-plugins/tez-yarn-timeline-history/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history/pom.xml @@ -146,7 +146,7 @@ <profiles> <profile> - <id>hadoop26</id> + <id>hadoop27</id> <dependencies> <dependency> <groupId>org.apache.tez</groupId> http://git-wip-us.apache.org/repos/asf/tez/blob/ca607d65/tez-tools/analyzers/pom.xml ---------------------------------------------------------------------- diff --git a/tez-tools/analyzers/pom.xml b/tez-tools/analyzers/pom.xml index 40542be..58f4ab7 100644 --- a/tez-tools/analyzers/pom.xml +++ b/tez-tools/analyzers/pom.xml @@ -27,16 +27,7 @@ <profiles> <profile> - <id>hadoop24</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>job-analyzer</module> - </modules> - </profile> - <profile> - <id>hadoop26</id> + <id>hadoop27</id> <activation> <property> <name>!skipATS</name>
