(TWILL-254) Update to use ContainerId.fromString This closes #65 on Github.
Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/twill/repo Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/36a4f118 Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/36a4f118 Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/36a4f118 Branch: refs/heads/branch-0.12.0 Commit: 36a4f1185ed41a28a5b30aeb836571beb7ffccb8 Parents: 1e97c74 Author: Clay Baenziger <[email protected]> Authored: Fri Feb 2 22:35:37 2018 -0500 Committer: Terence Yim <[email protected]> Committed: Fri Mar 16 14:09:58 2018 -0700 ---------------------------------------------------------------------- pom.xml | 51 ++++++++++++++++++-- .../internal/yarn/Hadoop20YarnAMClient.java | 7 +++ .../internal/yarn/Hadoop21YarnAMClient.java | 7 +++ .../internal/yarn/Hadoop22YarnAMClient.java | 2 +- .../internal/yarn/Hadoop26YarnAMClient.java | 42 ++++++++++++++++ .../internal/yarn/AbstractYarnAMClient.java | 12 +++-- .../yarn/VersionDetectYarnAMClientFactory.java | 10 +++- .../apache/twill/internal/yarn/YarnUtils.java | 13 ++++- 8 files changed, 132 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2ca9bb4..656d944 100644 --- a/pom.xml +++ b/pom.xml @@ -171,7 +171,6 @@ <junit.version>4.11</junit.version> <jopt-simple.version>3.2</jopt-simple.version> <commons-compress.version>1.5</commons-compress.version> - <hadoop.version>[2.0.2-alpha,2.3.0]</hadoop.version> <hadoop20.output.dir>target/hadoop20-classes</hadoop20.output.dir> </properties> @@ -538,9 +537,6 @@ <properties> <hadoop.version>2.3.0</hadoop.version> </properties> - <activation> - <activeByDefault>true</activeByDefault> - </activation> <build> <plugins> <plugin> @@ -723,6 +719,53 @@ </build> </profile> <profile> + <id>hadoop-2.6</id> + <properties> + <hadoop.version>2.6.5</hadoop.version> + </properties> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/main/hadoop21</source> + <source>src/main/hadoop22</source> + <source>src/main/hadoop23</source> + <source>src/main/hadoop26</source> + </sources> + </configuration> + </execution> + <execution> + <id>add-source-2.0</id> + <phase>prepare-package</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/main/hadoop20</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>java8-test</id> <modules> <module>twill-java8-test</module> http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java index a990cc0..67bef3e 100644 --- a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java +++ b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java @@ -26,11 +26,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.twill.internal.appmaster.RunnableProcessLauncher; import org.apache.twill.internal.yarn.ports.AMRMClient; import org.apache.twill.internal.yarn.ports.AMRMClientImpl; @@ -72,6 +74,11 @@ public final class Hadoop20YarnAMClient extends AbstractYarnAMClient<AMRMClient. } @Override + protected ContainerId containerIdLookup(String containerIdStr) { + return (ConverterUtils.toContainerId(containerIdStr)); + } + + @Override protected void startUp() throws Exception { Preconditions.checkNotNull(trackerAddr, "Tracker address not set."); Preconditions.checkNotNull(trackerUrl, "Tracker URL not set."); http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java index 82f428a..f349b4e 100644 --- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java +++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java @@ -27,11 +27,13 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.twill.internal.appmaster.RunnableProcessLauncher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +59,11 @@ public class Hadoop21YarnAMClient extends AbstractYarnAMClient<AMRMClient.Contai }; } + @Override + protected ContainerId containerIdLookup(String containerIdStr) { + return (ConverterUtils.toContainerId(containerIdStr)); + } + protected final AMRMClient<AMRMClient.ContainerRequest> amrmClient; protected final Hadoop21YarnNMClient nmClient; protected Resource maxCapability; http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java b/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java index 3ee99e8..c1bb5fd 100644 --- a/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java +++ b/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java @@ -26,7 +26,7 @@ import java.util.List; /** * Wrapper class for AMRMClient for Hadoop version 2.2 or greater. */ -public final class Hadoop22YarnAMClient extends Hadoop21YarnAMClient { +public class Hadoop22YarnAMClient extends Hadoop21YarnAMClient { private static final Logger LOG = LoggerFactory.getLogger(Hadoop22YarnAMClient.class); http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java b/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java new file mode 100644 index 0000000..372c422 --- /dev/null +++ b/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.internal.yarn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Wrapper class for AMRMClient for Hadoop version 2.6 or greater. + */ +public final class Hadoop26YarnAMClient extends Hadoop22YarnAMClient { + + private static final Logger LOG = LoggerFactory.getLogger(Hadoop26YarnAMClient.class); + + public Hadoop26YarnAMClient(Configuration conf) { + super(conf); + } + + @Override + protected final ContainerId containerIdLookup(String containerIdStr) { + return (ContainerId.fromString(containerIdStr)); + } +} http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java index e8f3b2b..b4b50cd 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java @@ -27,7 +27,6 @@ import com.google.common.util.concurrent.AbstractIdleService; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.twill.internal.ProcessLauncher; import org.apache.twill.internal.appmaster.RunnableProcessLauncher; import org.slf4j.Logger; @@ -80,7 +79,7 @@ public abstract class AbstractYarnAMClient<T> extends AbstractIdleService implem String masterContainerId = System.getenv().get(containerIdEnvName); Preconditions.checkArgument(masterContainerId != null, "Missing %s from environment", containerIdEnvName); - this.containerId = ConverterUtils.toContainerId(masterContainerId); + this.containerId = containerIdLookup(masterContainerId); this.inflightRequests = ArrayListMultimap.create(); this.pendingRequests = ArrayListMultimap.create(); this.pendingRemoves = Lists.newLinkedList(); @@ -89,7 +88,6 @@ public abstract class AbstractYarnAMClient<T> extends AbstractIdleService implem this.blacklistedResources = Lists.newArrayList(); } - @Override public final ContainerId getContainerId() { return containerId; @@ -227,6 +225,14 @@ public abstract class AbstractYarnAMClient<T> extends AbstractIdleService implem } /** + * Returns the ContainerId given a container ID string + * + * @param containerIdStr the container ID string to lookup + * @return A {@link ContainerId} instance representing the result. + */ + protected abstract ContainerId containerIdLookup(String containerIdStr); + + /** * Adjusts the given resource capability to fit in the cluster limit. * * @param capability The capability to be adjusted. http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java index 943efb8..c6ab02b 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java @@ -48,11 +48,17 @@ public final class VersionDetectYarnAMClientFactory implements YarnAMClientFacto clzName = getClass().getPackage().getName() + ".Hadoop21YarnAMClient"; clz = (Class<YarnAMClient>) Class.forName(clzName); break; - default: - // Uses hadoop-2.2 or above class + case HADOOP_22: + case HADOOP_23: + // Uses hadoop-2.2 class clzName = getClass().getPackage().getName() + ".Hadoop22YarnAMClient"; clz = (Class<YarnAMClient>) Class.forName(clzName); break; + default: + // Uses hadoop-2.6 or above class + clzName = getClass().getPackage().getName() + ".Hadoop26YarnAMClient"; + clz = (Class<YarnAMClient>) Class.forName(clzName); + break; } return clz.getConstructor(Configuration.class).newInstance(conf); http://git-wip-us.apache.org/repos/asf/twill/blob/36a4f118/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java index c0aeb0c..d7e6eb0 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java @@ -68,7 +68,8 @@ public class YarnUtils { HADOOP_20, HADOOP_21, HADOOP_22, - HADOOP_23 + HADOOP_23, + HADOOP_26 } private static final Logger LOG = LoggerFactory.getLogger(YarnUtils.class); @@ -263,7 +264,15 @@ public class YarnUtils { Class.forName("org.apache.hadoop.yarn.client.cli.LogsCLI"); try { Class.forName("org.apache.hadoop.yarn.conf.HAUtil"); - HADOOP_VERSION.set(HadoopVersions.HADOOP_23); + try { + Class[] args = new Class[1]; + args[0] = String.class; + // see if we have a org.apache.hadoop.yarn.api.records.ContainerId.fromString() method + Class.forName("org.apache.hadoop.yarn.api.records.ContainerId").getMethod("fromString", args); + HADOOP_VERSION.set(HadoopVersions.HADOOP_26); + } catch (NoSuchMethodException e) { + HADOOP_VERSION.set(HadoopVersions.HADOOP_23); + } } catch (ClassNotFoundException e) { HADOOP_VERSION.set(HadoopVersions.HADOOP_22); }
