Repository: tez Updated Branches: refs/heads/master 24ba80f81 -> 40e864d14
TEZ_2928. Framework for Hadoop shims. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/40e864d1 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/40e864d1 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/40e864d1 Branch: refs/heads/master Commit: 40e864d145458ce94588b90f065204d80ffa1cc7 Parents: 24ba80f Author: Hitesh Shah <[email protected]> Authored: Fri Nov 13 16:27:47 2015 -0800 Committer: Hitesh Shah <[email protected]> Committed: Fri Nov 13 16:27:47 2015 -0800 ---------------------------------------------------------------------- BUILDING.txt | 3 + CHANGES.txt | 1 + hadoop-shim-impls/findbugs-exclude.xml | 16 +++ .../hadoop-shim-2.4/findbugs-exclude.xml | 16 +++ hadoop-shim-impls/hadoop-shim-2.4/pom.xml | 50 ++++++++ .../hadoop/shim/HadoopShim23_24Provider.java | 33 +++++ .../apache/tez/hadoop/shim/HadoopShim24.java | 45 +++++++ ...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 +++ .../shim/TestHadoop23_24ShimProvider.java | 38 ++++++ .../hadoop-shim-2.6/findbugs-exclude.xml | 16 +++ hadoop-shim-impls/hadoop-shim-2.6/pom.xml | 50 ++++++++ .../hadoop/shim/HadoopShim25_26_27Provider.java | 33 +++++ .../apache/tez/hadoop/shim/HadoopShim26.java | 52 ++++++++ ...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 +++ .../shim/TestHadoop25_26_27ShimProvider.java | 39 ++++++ .../hadoop-shim-2.8/findbugs-exclude.xml | 16 +++ hadoop-shim-impls/hadoop-shim-2.8/pom.xml | 50 ++++++++ .../apache/tez/hadoop/shim/HadoopShim28.java | 52 ++++++++ .../tez/hadoop/shim/HadoopShim28Provider.java | 30 +++++ ...rg.apache.tez.hadoop.shim.HadoopShimProvider | 14 +++ .../hadoop/shim/TestHadoopShim28Provider.java | 38 ++++++ hadoop-shim-impls/pom.xml | 58 +++++++++ hadoop-shim/findbugs-exclude.xml | 16 +++ hadoop-shim/pom.xml | 46 +++++++ .../tez/hadoop/shim/DefaultHadoopShim.java | 28 +++++ .../org/apache/tez/hadoop/shim/HadoopShim.java | 60 +++++++++ .../tez/hadoop/shim/HadoopShimProvider.java | 38 ++++++ .../tez/hadoop/shim/HadoopShimsLoader.java | 126 +++++++++++++++++++ pom.xml | 7 ++ tez-dag/pom.xml | 4 + .../java/org/apache/tez/dag/app/AppContext.java | 3 + .../org/apache/tez/dag/app/DAGAppMaster.java | 10 ++ 32 files changed, 1016 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt index 7e4d39a..241b7a8 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -89,6 +89,9 @@ For recent versions of Hadoop, you can bundle AWS-S3 (2.6.0+) or Azure (2.7.0+) $ mvn package -Dhadoop.version=${X} -Paws -Pazure +Tez also has some shims to provide version-specific implementations for various APIs. +For more details, please refer to https://cwiki.apache.org/confluence/display/TEZ/HadoopShims + For building on Windows with the hadoop24 profile enabled: Tez has a couple of symlinks in its codebase. Windows is a bit difficult with http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7233b03..399c6d5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-2679. Admin forms of launch env settings ALL CHANGES: + TEZ-2928. Framework for Hadoop shims. TEZ-2935. Add MR slow start translation for ShuffleVertexManager TEZ-2918. Make progress notifications in IOs TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/findbugs-exclude.xml b/hadoop-shim-impls/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/hadoop-shim-impls/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml new file mode 100644 index 0000000..095c2b1 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hadoop-shim-impls</artifactId> + <groupId>org.apache.tez</groupId> + <version>0.8.2-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>hadoop-shim-2.4</artifactId> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java new file mode 100644 index 0000000..d94b5bf --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +@Private +public class HadoopShim23_24Provider extends HadoopShimProvider { + + @Override + public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { + if (majorVersion == 2 && (minorVersion == 3 || minorVersion == 4)) { + return new org.apache.tez.hadoop.shim.HadoopShim24(); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java new file mode 100644 index 0000000..3f7e56e --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; + +@Private +public class HadoopShim24 extends HadoopShim { + + @Override + public void setHadoopCallerContext(String context) { + // Not supported + } + + @Override + public void clearHadoopCallerContext() { + // Not supported + } + + @Override + public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + // Not supported + return null; + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider new file mode 100644 index 0000000..a675e20 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.tez.hadoop.shim.HadoopShim23_24Provider http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java new file mode 100644 index 0000000..9bc4435 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.junit.Assert; +import org.junit.Test; + +public class TestHadoop23_24ShimProvider { + + @Test + public void testShimProvider() { + HadoopShim23_24Provider provider = new HadoopShim23_24Provider(); + Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 5)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 6)); + Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4)); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml new file mode 100644 index 0000000..17f4711 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hadoop-shim-impls</artifactId> + <groupId>org.apache.tez</groupId> + <version>0.8.2-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>hadoop-shim-2.6</artifactId> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java new file mode 100644 index 0000000..ab013e9 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +@Private +public class HadoopShim25_26_27Provider extends HadoopShimProvider { + + @Override + public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { + if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) { + return new HadoopShim26(); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java new file mode 100644 index 0000000..c1622dc --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; + +@Private +public class HadoopShim26 extends HadoopShim { + + @Override + public void setHadoopCallerContext(String context) { + // Not supported + } + + @Override + public void clearHadoopCallerContext() { + // Not supported + } + + @Override + public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes(); + Set<String> supportedTypes = new HashSet<String>(); + for (SchedulerResourceTypes resourceType : supportedResourceTypes) { + supportedTypes.add(resourceType.name()); + } + return supportedTypes; + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider new file mode 100644 index 0000000..4226f18 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java new file mode 100644 index 0000000..0e6d997 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.junit.Assert; +import org.junit.Test; + +public class TestHadoop25_26_27ShimProvider { + + @Test + public void testShimProvider() { + HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider(); + Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 3)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 8)); + Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7)); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml new file mode 100644 index 0000000..18d22c5 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hadoop-shim-impls</artifactId> + <groupId>org.apache.tez</groupId> + <version>0.8.2-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>hadoop-shim-2.8</artifactId> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java new file mode 100644 index 0000000..0c599e4 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; + +public class HadoopShim28 extends HadoopShim { + + final static CallerContext nullCallerContext = new CallerContext.Builder("").build(); + + @Override + public void setHadoopCallerContext(String context) { + CallerContext.setCurrent(new CallerContext.Builder(context).build()); + } + + @Override + public void clearHadoopCallerContext() { + CallerContext.setCurrent(nullCallerContext); + } + + @Override + public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes(); + Set<String> supportedTypes = new HashSet<String>(); + for (SchedulerResourceTypes resourceType : supportedResourceTypes) { + supportedTypes.add(resourceType.name()); + } + return supportedTypes; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java new file mode 100644 index 0000000..d03a538 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +public class HadoopShim28Provider extends HadoopShimProvider { + + @Override + public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { + if (majorVersion == 2 && minorVersion >= 8) { + return new HadoopShim28(); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider new file mode 100644 index 0000000..24fe49d --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.tez.hadoop.shim.HadoopShim28Provider http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java new file mode 100644 index 0000000..df92627 --- /dev/null +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.junit.Assert; +import org.junit.Test; + +public class TestHadoopShim28Provider { + + @Test + public void testShimProvider() { + HadoopShim28Provider provider = new HadoopShim28Provider(); + Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 7)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 5)); + Assert.assertNull(provider.createHadoopShim("foo", 2, 6)); + Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8)); + Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111)); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim-impls/pom.xml b/hadoop-shim-impls/pom.xml new file mode 100644 index 0000000..6c8f7d2 --- /dev/null +++ b/hadoop-shim-impls/pom.xml @@ -0,0 +1,58 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>tez</artifactId> + <groupId>org.apache.tez</groupId> + <version>0.8.2-SNAPSHOT</version> + </parent> + <artifactId>hadoop-shim-impls</artifactId> + <packaging>pom</packaging> + + <profiles> + <profile> + <id>hadoop24</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <modules> + <module>hadoop-shim-2.4</module> + </modules> + </profile> + <profile> + <id>hadoop26</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <modules> + <module>hadoop-shim-2.6</module> + </modules> + </profile> + <profile> + <id>hadoop28</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <modules> + <module>hadoop-shim-2.8</module> + </modules> + </profile> + </profiles> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim/findbugs-exclude.xml b/hadoop-shim/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/hadoop-shim/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-shim/pom.xml b/hadoop-shim/pom.xml new file mode 100644 index 0000000..1f70e6f --- /dev/null +++ b/hadoop-shim/pom.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>tez</artifactId> + <groupId>org.apache.tez</groupId> + <version>0.8.2-SNAPSHOT</version> + </parent> + <artifactId>hadoop-shim</artifactId> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java new file mode 100644 index 0000000..c68777c --- /dev/null +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +/** + * Default Hadoop Shim. For now, it mimics HadoopShim. + */ +@Private +public class DefaultHadoopShim extends HadoopShim { +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java new file mode 100644 index 0000000..7314c2d --- /dev/null +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; + +import com.google.common.collect.Sets; + +@Public +@Unstable +public abstract class HadoopShim { + + /** + * Set up Hadoop Caller Context + * @param context Context to be set + */ + public void setHadoopCallerContext(String context) { + // Nothing to do + } + + /** + * Clear the Hadoop Caller Context + */ + public void clearHadoopCallerContext() { + // Nothing to do + } + + public static String CPU_RESOURCE = "CPU"; + public static String MEMORY_RESOURCE = "MEMORY"; + + /** + * Extract supported Resource types from the RM's response when the AM registers + * @param response ApplicationMasterResponse from RM after registering + * @return Set of Resource types that are supported + */ + public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + return null; + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java new file mode 100644 index 0000000..728408c --- /dev/null +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Public +@Unstable +public abstract class HadoopShimProvider { + + /** + * Provides a HadoopShim if the version matches the implementation of the Shim + * @param hadoopVersion Full version string of hadoop-common's version + * @param majorVersion Major version + * @param minorVersion Minor version + * @return HadoopShim object if there is a match else null + */ + public abstract HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, + int minorVersion); + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java new file mode 100644 index 0000000..fc2ec81 --- /dev/null +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.hadoop.shim; + +import java.util.ServiceLoader; +import java.util.StringTokenizer; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.util.VersionInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Private +public class HadoopShimsLoader { + + private static final Logger LOG = LoggerFactory.getLogger(HadoopShimsLoader.class); + + private static ServiceLoader<HadoopShimProvider> shimLoader = + ServiceLoader.load(HadoopShimProvider.class); + + private final HadoopShim currentShim; + private final HadoopShimProvider currentShimProvider; + + public HadoopShimsLoader() { + String versionStr = VersionInfo.getVersion(); + Version version = new Version(versionStr); + HadoopShim selectedShim = null; + HadoopShimProvider selectedShimProvider = null; + LOG.info("Trying to locate HadoopShimProvider for " + + "hadoopVersion=" + versionStr + + ", majorVersion=" + version.majorVersion + + ", minorVersion=" + version.minorVersion); + synchronized (shimLoader) { + for (HadoopShimProvider provider : shimLoader) { + LOG.debug("Trying HadoopShimProvider : " + + provider.getClass().getName()); + HadoopShim hadoopShim = null; + try { + hadoopShim = provider.createHadoopShim(versionStr, + version.majorVersion, version.minorVersion); + + if (hadoopShim != null) { + selectedShim = hadoopShim; + selectedShimProvider = provider; + break; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot pick " + provider.getClass().getName() + + " as the HadoopShimProvider - returned null hadoop shim"); + } + } + } catch (Exception e) { + LOG.info("Failed to use " + provider.getClass().getName() + + " due to error: ", e); + } + } + if (selectedShim == null) { + currentShim = new DefaultHadoopShim(); + currentShimProvider = null; + } else { + currentShim = selectedShim; + currentShimProvider = selectedShimProvider; + } + } + LOG.info("Picked HadoopShim " + currentShim.getClass().getName() + + ", providerName=" + + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null" ) + + ", hadoopVersion=" + versionStr + + ", majorVersion=" + version.majorVersion + + ", minorVersion=" + version.minorVersion); + + } + + public HadoopShim getHadoopShim() { + return currentShim; + } + + private static class Version { + + int majorVersion = -1; + int minorVersion = -1; + + public Version(String versionString) { + int index = 0; + StringTokenizer tokenizer = new StringTokenizer(versionString, ".-", true); + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + if (token.equals("-")) { + break; + } + if (token.equals(".")) { + continue; + } + try { + int i = Integer.parseInt(token); + if (index == 0) { + majorVersion = i; + } else if (index == 1) { + minorVersion = i; + break; + } + ++index; + } catch (NumberFormatException nfe) { + break; + } + } + } + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b8b8a22..a17f714 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,11 @@ <dependencies> <dependency> <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> <artifactId>tez-api</artifactId> <version>${project.version}</version> </dependency> @@ -679,6 +684,7 @@ </dependencyManagement> <modules> + <module>hadoop-shim</module> <module>tez-api</module> <module>tez-common</module> <module>tez-runtime-library</module> @@ -691,6 +697,7 @@ <module>tez-ui</module> <module>tez-plugins</module> <module>tez-tools</module> + <module>hadoop-shim-impls</module> <module>tez-dist</module> <module>docs</module> </modules> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/pom.xml ---------------------------------------------------------------------- diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index adda218..7ca92db 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -35,6 +35,10 @@ <dependencies> <dependency> <groupId>org.apache.tez</groupId> + <artifactId>hadoop-shim</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> <artifactId>tez-api</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java index 657267b..68453b1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java @@ -37,6 +37,7 @@ import org.apache.tez.dag.app.rm.node.AMNodeTracker; import org.apache.tez.common.security.ACLManager; import org.apache.tez.dag.history.HistoryEventHandler; import org.apache.tez.dag.records.TezDAGID; +import org.apache.tez.hadoop.shim.HadoopShim; import com.google.common.util.concurrent.ListeningExecutorService; @@ -123,4 +124,6 @@ public interface AppContext { public String getTaskSchedulerName(int schedulerId); public String getContainerLauncherName(int launcherId); + public HadoopShim getHadoopShim(); + } http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index f4c57e3..9c09f95 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -174,6 +174,8 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.dag.utils.Graph; import org.apache.tez.dag.utils.RelocalizationUtils; import org.apache.tez.dag.utils.Simple2LevelVersionComparator; +import org.apache.tez.hadoop.shim.HadoopShim; +import org.apache.tez.hadoop.shim.HadoopShimsLoader; import org.apache.tez.util.TezMxBeanResourceCalculator; import org.codehaus.jettison.json.JSONException; import org.slf4j.Logger; @@ -234,6 +236,7 @@ public class DAGAppMaster extends AbstractService { private final String[] localDirs; private final String[] logDirs; private final AMPluginDescriptorProto amPluginDescriptorProto; + private final HadoopShim hadoopShim; private ContainerSignatureMatcher containerSignatureMatcher; private AMContainerMap containers; private AMNodeTracker nodes; @@ -353,9 +356,11 @@ public class DAGAppMaster extends AbstractService { this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort, this.containerID.toString(), this.appMasterUgi.getShortUserName()); + this.hadoopShim = new HadoopShimsLoader().getHadoopShim(); LOG.info("Created DAGAppMaster for application " + applicationAttemptId + ", versionInfo=" + dagVersionInfo.toString()); + } // Pull this WebAppUtils function into Tez until YARN-4186 @@ -1596,6 +1601,11 @@ public class DAGAppMaster extends AbstractService { } @Override + public HadoopShim getHadoopShim() { + return hadoopShim; + } + + @Override public Map<ApplicationAccessType, String> getApplicationACLs() { if (getServiceState() != STATE.STARTED) { throw new TezUncheckedException(
