(TWILL-207) Only use list of class names as the cache name - Also some indentation changes.
This closes #35 on Github. Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/twill/repo Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/0cc3159f Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/0cc3159f Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/0cc3159f Branch: refs/heads/site Commit: 0cc3159fe529b93cb33f8cca3d2ba37b0012a455 Parents: f4df32d Author: Terence Yim <[email protected]> Authored: Tue Feb 28 18:03:45 2017 -0800 Committer: Terence Yim <[email protected]> Committed: Thu Mar 2 22:56:41 2017 -0800 ---------------------------------------------------------------------- .../apache/twill/yarn/YarnTwillPreparer.java | 123 ++++++++++--------- 1 file changed, 63 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/twill/blob/0cc3159f/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java index 07594e3..6fbbf93 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java @@ -186,7 +186,7 @@ final class YarnTwillPreparer implements TwillPreparer { private void confirmRunnableName(String runnableName) { Preconditions.checkNotNull(runnableName); Preconditions.checkArgument(twillSpec.getRunnables().containsKey(runnableName), - "Runnable %s is not defined in the application.", runnableName); + "Runnable %s is not defined in the application.", runnableName); } @Override @@ -369,57 +369,59 @@ final class YarnTwillPreparer implements TwillPreparer { final ApplicationMasterInfo appMasterInfo = launcher.getContainerInfo(); Callable<ProcessController<YarnApplicationReport>> submitTask = new Callable<ProcessController<YarnApplicationReport>>() { - @Override - public ProcessController<YarnApplicationReport> call() throws Exception { - - // Local files needed by AM - Map<String, LocalFile> localFiles = Maps.newHashMap(); - - createLauncherJar(localFiles); - createTwillJar(createBundler(classAcceptor), localFiles); - createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles); - createResourcesJar(createBundler(classAcceptor), localFiles); - - Path runtimeConfigDir = Files.createTempDirectory(localStagingDir.toPath(), - Constants.Files.RUNTIME_CONFIG_JAR); - try { - saveSpecification(twillSpec, runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC)); - saveLogback(runtimeConfigDir.resolve(Constants.Files.LOGBACK_TEMPLATE)); - saveClassPaths(runtimeConfigDir); - saveJvmOptions(runtimeConfigDir.resolve(Constants.Files.JVM_OPTIONS)); - saveArguments(new Arguments(arguments, runnableArgs), runtimeConfigDir.resolve(Constants.Files.ARGUMENTS)); - saveEnvironments(runtimeConfigDir.resolve(Constants.Files.ENVIRONMENTS)); - createRuntimeConfigJar(runtimeConfigDir, localFiles); - } finally { - Paths.deleteRecursively(runtimeConfigDir); - } + @Override + public ProcessController<YarnApplicationReport> call() throws Exception { + + // Local files needed by AM + Map<String, LocalFile> localFiles = Maps.newHashMap(); + + createLauncherJar(localFiles); + createTwillJar(createBundler(classAcceptor), localFiles); + createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles); + createResourcesJar(createBundler(classAcceptor), localFiles); + + Path runtimeConfigDir = Files.createTempDirectory(localStagingDir.toPath(), + Constants.Files.RUNTIME_CONFIG_JAR); + try { + saveSpecification(twillSpec, runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC)); + saveLogback(runtimeConfigDir.resolve(Constants.Files.LOGBACK_TEMPLATE)); + saveClassPaths(runtimeConfigDir); + saveJvmOptions(runtimeConfigDir.resolve(Constants.Files.JVM_OPTIONS)); + saveArguments(new Arguments(arguments, runnableArgs), + runtimeConfigDir.resolve(Constants.Files.ARGUMENTS)); + saveEnvironments(runtimeConfigDir.resolve(Constants.Files.ENVIRONMENTS)); + createRuntimeConfigJar(runtimeConfigDir, localFiles); + } finally { + Paths.deleteRecursively(runtimeConfigDir); + } - createLocalizeFilesJson(localFiles); - - LOG.debug("Submit AM container spec: {}", appMasterInfo); - // java -Djava.io.tmpdir=tmp -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory - // org.apache.twill.internal.TwillLauncher - // appMaster.jar - // org.apache.twill.internal.appmaster.ApplicationMasterMain - // false - - int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), - Constants.APP_MASTER_RESERVED_MEMORY_MB, Constants.HEAP_MIN_RATIO); - return launcher.prepareLaunch(ImmutableMap.<String, String>of(), localFiles.values(), credentials) - .addCommand( - "$JAVA_HOME/bin/java", - "-Djava.io.tmpdir=tmp", - "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, - "-Dtwill.app=$" + Constants.TWILL_APP_NAME, - "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", - "-Xmx" + memory + "m", - extraOptions == null ? "" : extraOptions, - TwillLauncher.class.getName(), - ApplicationMasterMain.class.getName(), - Boolean.FALSE.toString()) - .launch(); - } - }; + createLocalizeFilesJson(localFiles); + + LOG.debug("Submit AM container spec: {}", appMasterInfo); + // java -Djava.io.tmpdir=tmp -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory + // org.apache.twill.internal.TwillLauncher + // appMaster.jar + // org.apache.twill.internal.appmaster.ApplicationMasterMain + // false + + int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), + Constants.APP_MASTER_RESERVED_MEMORY_MB, + Constants.HEAP_MIN_RATIO); + return launcher.prepareLaunch(ImmutableMap.<String, String>of(), localFiles.values(), credentials) + .addCommand( + "$JAVA_HOME/bin/java", + "-Djava.io.tmpdir=tmp", + "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, + "-Dtwill.app=$" + Constants.TWILL_APP_NAME, + "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", + "-Xmx" + memory + "m", + extraOptions == null ? "" : extraOptions, + TwillLauncher.class.getName(), + ApplicationMasterMain.class.getName(), + Boolean.FALSE.toString()) + .launch(); + } + }; YarnTwillController controller = controllerFactory.create(runId, logHandlers, submitTask, timeout, timeoutUnit); controller.start(); @@ -487,7 +489,7 @@ final class YarnTwillPreparer implements TwillPreparer { public void load(String name, Location targetLocation) throws IOException { // Stuck in the yarnAppClient class to make bundler being able to pickup the right yarn-client version bundler.createBundle(targetLocation, ApplicationMasterMain.class, - yarnAppClient.getClass(), TwillContainerMain.class, OptionSpec.class); + yarnAppClient.getClass(), TwillContainerMain.class, OptionSpec.class); } }); @@ -519,7 +521,8 @@ final class YarnTwillPreparer implements TwillPreparer { for (String name : classList) { hasher.putString(name); } - String name = twillSpec.getName() + "-" + hasher.hash().toString() + "-" + Constants.Files.APPLICATION_JAR; + // Only depends on class list so that it can be reused across different launches + String name = hasher.hash().toString() + "-" + Constants.Files.APPLICATION_JAR; LOG.debug("Create and copy {}", Constants.Files.APPLICATION_JAR); Location location = locationCache.get(name, new LocationCache.Loader() { @@ -612,12 +615,12 @@ final class YarnTwillPreparer implements TwillPreparer { // Rewrite LocalFiles inside twillSpec Map<String, RuntimeSpecification> runtimeSpec = Maps.transformEntries( spec.getRunnables(), new Maps.EntryTransformer<String, RuntimeSpecification, RuntimeSpecification>() { - @Override - public RuntimeSpecification transformEntry(String key, RuntimeSpecification value) { - return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), - value.getResourceSpecification(), runnableLocalFiles.get(key)); - } - }); + @Override + public RuntimeSpecification transformEntry(String key, RuntimeSpecification value) { + return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), + value.getResourceSpecification(), runnableLocalFiles.get(key)); + } + }); // Serialize into a local temp file. LOG.debug("Creating {}", targetFile); @@ -661,7 +664,7 @@ final class YarnTwillPreparer implements TwillPreparer { @Override public void load(String name, Location targetLocation) throws IOException { // Create a jar file with the TwillLauncher and FindFreePort and dependent classes inside. - try (final JarOutputStream jarOut = new JarOutputStream(targetLocation.getOutputStream())) { + try (JarOutputStream jarOut = new JarOutputStream(targetLocation.getOutputStream())) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { classLoader = getClass().getClassLoader();
