(TWILL-228) Remove the optimization when building application jar - The optimization can leading skipping certain jars in the application jar
This closes #50 on Github. Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/twill/repo Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/2a316a60 Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/2a316a60 Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/2a316a60 Branch: refs/heads/site Commit: 2a316a60b041f39bff95bfcd45f308f7c89d4bfa Parents: e61be38 Author: Terence Yim <[email protected]> Authored: Fri Mar 31 15:39:20 2017 -0700 Committer: Terence Yim <[email protected]> Committed: Fri Mar 31 16:15:14 2017 -0700 ---------------------------------------------------------------------- .../apache/twill/yarn/YarnTwillPreparer.java | 23 +++----------------- .../twill/yarn/YarnTwillRunnerService.java | 21 +----------------- 2 files changed, 4 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/twill/blob/2a316a60/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java index 4846fe3..c8abf4f 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java @@ -146,7 +146,6 @@ final class YarnTwillPreparer implements TwillPreparer { private final Credentials credentials; private final Map<String, Map<String, String>> logLevels = Maps.newHashMap(); private final LocationCache locationCache; - private final Set<URL> twillClassPaths; private String schedulerQueue; private String extraOptions; private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG; @@ -154,8 +153,8 @@ final class YarnTwillPreparer implements TwillPreparer { private final Map<String, Integer> maxRetries = Maps.newHashMap(); YarnTwillPreparer(Configuration config, TwillSpecification twillSpec, RunId runId, - String zkConnectString, Location appLocation, Set<URL> twillClassPaths, - String extraOptions, LocationCache locationCache, YarnTwillControllerFactory controllerFactory) { + String zkConnectString, Location appLocation, String extraOptions, + LocationCache locationCache, YarnTwillControllerFactory controllerFactory) { this.config = config; this.twillSpec = twillSpec; this.runId = runId; @@ -166,7 +165,6 @@ final class YarnTwillPreparer implements TwillPreparer { this.extraOptions = extraOptions; this.classAcceptor = new ClassAcceptor(); this.locationCache = locationCache; - this.twillClassPaths = twillClassPaths; } private void confirmRunnableName(String runnableName) { @@ -372,7 +370,7 @@ final class YarnTwillPreparer implements TwillPreparer { createLauncherJar(localFiles); createTwillJar(createBundler(classAcceptor), yarnAppClient, localFiles); - createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles); + createApplicationJar(createBundler(classAcceptor), localFiles); createResourcesJar(createBundler(classAcceptor), localFiles); Path runtimeConfigDir = Files.createTempDirectory(getLocalStagingDir().toPath(), @@ -835,19 +833,4 @@ final class YarnTwillPreparer implements TwillPreparer { private ApplicationBundler createBundler(ClassAcceptor classAcceptor) { return new ApplicationBundler(classAcceptor).setTempDir(getLocalStagingDir()); } - - /** - * Creates a {@link ApplicationBundler} for building application jar. The bundler will include classes - * accepted by the given {@link ClassAcceptor}, as long as it is not a twill class. - */ - private ApplicationBundler createApplicationJarBundler(final ClassAcceptor classAcceptor) { - // Accept classes based on the classAcceptor and also excluding all twill classes as they are already - // in the twill.jar - return createBundler(new ClassAcceptor() { - @Override - public boolean accept(String className, URL classUrl, URL classPathUrl) { - return !twillClassPaths.contains(classPathUrl) && classAcceptor.accept(className, classUrl, classPathUrl); - } - }); - } } http://git-wip-us.apache.org/repos/asf/twill/blob/2a316a60/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java index 269ffdf..20627e2 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.twill.api.ClassAcceptor; import org.apache.twill.api.Configs; import org.apache.twill.api.LocalFile; import org.apache.twill.api.ResourceSpecification; @@ -73,7 +72,6 @@ import org.apache.twill.internal.appmaster.ApplicationMasterLiveNodeData; import org.apache.twill.internal.io.BasicLocationCache; import org.apache.twill.internal.io.LocationCache; import org.apache.twill.internal.io.NoCachingLocationCache; -import org.apache.twill.internal.utils.Dependencies; import org.apache.twill.internal.yarn.VersionDetectYarnAppClientFactory; import org.apache.twill.internal.yarn.YarnAppClient; import org.apache.twill.internal.yarn.YarnApplicationReport; @@ -96,7 +94,6 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URL; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -134,7 +131,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService { private final ZKClientService zkClientService; private final LocationFactory locationFactory; private final Table<String, RunId, YarnTwillController> controllers; - private final Set<URL> twillClassPaths; // A Guava service to help the state transition. private final Service serviceDelegate; private LocationCache locationCache; @@ -169,7 +165,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService { this.locationFactory = locationFactory; this.zkClientService = getZKClientService(zkConnect); this.controllers = HashBasedTable.create(); - this.twillClassPaths = new HashSet<>(); this.serviceDelegate = new AbstractIdleService() { @Override protected void startUp() throws Exception { @@ -308,8 +303,7 @@ public final class YarnTwillRunnerService implements TwillRunnerService { Configuration config = new Configuration(yarnConfig); return new YarnTwillPreparer(config, twillSpec, runId, zkClientService.getConnectString(), - appLocation, twillClassPaths, jvmOptions, - locationCache, new YarnTwillControllerFactory() { + appLocation, jvmOptions, locationCache, new YarnTwillControllerFactory() { @Override public YarnTwillController create(RunId runId, boolean logCollectionEnabled, Iterable<LogHandler> logHandlers, Callable<ProcessController<YarnApplicationReport>> startUp, @@ -355,19 +349,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService { private void startUp() throws Exception { zkClientService.startAndWait(); - // Find all the classpaths for Twill classes. It is used for class filtering when building application jar - // in the YarnTwillPreparer - Dependencies.findClassDependencies(getClass().getClassLoader(), new ClassAcceptor() { - @Override - public boolean accept(String className, URL classUrl, URL classPathUrl) { - if (!className.startsWith("org.apache.twill.")) { - return false; - } - twillClassPaths.add(classPathUrl); - return true; - } - }, getClass().getName()); - // Create the root node, so that the namespace root would get created if it is missing // If the exception is caused by node exists, then it's ok. Otherwise propagate the exception. ZKOperations.ignoreError(zkClientService.create("/", null, CreateMode.PERSISTENT),
