(TWILL-179) Added support for custom ClassLoader for containers - Added method TwillPreparer.setClassLoader - Use system property "twill.custom.class.loader" to pass the class name of the custom ClassLoader
This closes #51 on Github. Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/twill/repo Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/c8e2a615 Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/c8e2a615 Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/c8e2a615 Branch: refs/heads/site Commit: c8e2a615a2450c85e9344b50ee9ded562a54d018 Parents: 2a316a6 Author: Terence Yim <[email protected]> Authored: Mon Apr 3 12:34:14 2017 -0700 Committer: Terence Yim <[email protected]> Committed: Mon Apr 3 13:12:38 2017 -0700 ---------------------------------------------------------------------- .../org/apache/twill/api/TwillPreparer.java | 13 +++ .../org/apache/twill/internal/Constants.java | 5 ++ .../apache/twill/launcher/TwillLauncher.java | 44 +++++++--- .../apache/twill/yarn/YarnTwillPreparer.java | 32 +++++-- .../apache/twill/yarn/CustomClassLoader.java | 87 ++++++++++++++++++++ .../twill/yarn/CustomClassLoaderRunnable.java | 56 +++++++++++++ .../twill/yarn/CustomClassLoaderTestRun.java | 42 ++++++++++ .../org/apache/twill/yarn/YarnTestSuite.java | 1 + 8 files changed, 264 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java ---------------------------------------------------------------------- diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java index 43b751b..1f50972 100644 --- a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java +++ b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java @@ -21,6 +21,7 @@ import org.apache.twill.api.logging.LogEntry; import org.apache.twill.api.logging.LogHandler; import java.net.URI; +import java.net.URL; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -276,6 +277,18 @@ public interface TwillPreparer { TwillPreparer setLogLevels(String runnableName, Map<String, LogEntry.Level> logLevelsForRunnable); /** + * Sets the class name of the {@link ClassLoader} to be used for loading twill and application classes for + * all containers. The {@link ClassLoader} class should have a public constructor that takes two parameters in the + * form of {@code (URL[] urls, ClassLoader parentClassLoader)}. + * The first parameter is an array of {@link URL} that contains the list of {@link URL} for loading classes and + * resources; the second parameter is the parent {@link ClassLoader}. + * + * @param classLoaderClassName name of the {@link ClassLoader} class. + * @return This {@link TwillPreparer}. + */ + TwillPreparer setClassLoader(String classLoaderClassName); + + /** * Starts the application. It's the same as calling {@link #start(long, TimeUnit)} with timeout of 60 seconds. * * @return A {@link TwillController} for controlling the running application. http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-common/src/main/java/org/apache/twill/internal/Constants.java ---------------------------------------------------------------------- diff --git a/twill-common/src/main/java/org/apache/twill/internal/Constants.java b/twill-common/src/main/java/org/apache/twill/internal/Constants.java index 6e799d5..4135c9a 100644 --- a/twill-common/src/main/java/org/apache/twill/internal/Constants.java +++ b/twill-common/src/main/java/org/apache/twill/internal/Constants.java @@ -53,6 +53,11 @@ public final class Constants { public static final String TWILL_APP_NAME = "TWILL_APP_NAME"; /** + * Constant for the system property name that carries the class name for the container ClassLoader as defined by user. + */ + public static final String TWILL_CONTAINER_CLASSLOADER = "twill.container.class.loader"; + + /** * Constants for names of internal files that are shared between client, AM and containers. */ public static final class Files { http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java ---------------------------------------------------------------------- diff --git a/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java b/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java index a5052b3..056639f 100644 --- a/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java +++ b/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java @@ -22,13 +22,11 @@ import org.apache.twill.internal.Constants; import java.io.BufferedReader; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; @@ -61,11 +59,12 @@ public final class TwillLauncher { boolean userClassPath = Boolean.parseBoolean(args[1]); // Create ClassLoader - URLClassLoader classLoader = createClassLoader(userClassPath); - Thread.currentThread().setContextClassLoader(classLoader); + URL[] classpath = createClasspath(userClassPath); + ClassLoader classLoader = createContainerClassLoader(classpath); + System.out.println("Launch class (" + mainClassName + ") using classloader " + classLoader.getClass().getName() + + " with classpath: " + Arrays.toString(classpath)); - System.out.println("Launch class (" + mainClassName + ") with classpath: " + - Arrays.toString(classLoader.getURLs())); + Thread.currentThread().setContextClassLoader(classLoader); Class<?> mainClass = classLoader.loadClass(mainClassName); Method mainMethod = mainClass.getMethod("main", String[].class); @@ -77,7 +76,7 @@ public final class TwillLauncher { System.out.println("Launcher completed"); } - private static URLClassLoader createClassLoader(boolean useClassPath) throws Exception { + private static URL[] createClasspath(boolean useClassPath) throws IOException { List<URL> urls = new ArrayList<>(); File appJarDir = new File(Constants.Files.APPLICATION_JAR); @@ -116,7 +115,33 @@ public final class TwillLauncher { } addClassPathsToList(urls, new File(Constants.Files.RUNTIME_CONFIG_JAR, Constants.Files.APPLICATION_CLASSPATH)); - return new URLClassLoader(urls.toArray(new URL[urls.size()])); + return urls.toArray(new URL[urls.size()]); + } + + /** + * Creates a {@link ClassLoader} to be used by this container that load classes from the given classpath. + */ + private static ClassLoader createContainerClassLoader(URL[] classpath) { + String containerClassLoaderName = System.getProperty(Constants.TWILL_CONTAINER_CLASSLOADER); + URLClassLoader classLoader = new URLClassLoader(classpath); + if (containerClassLoaderName == null) { + return classLoader; + } + + try { + @SuppressWarnings("unchecked") + Class<? extends ClassLoader> cls = (Class<? extends ClassLoader>) classLoader.loadClass(containerClassLoaderName); + + // Instantiate with constructor (URL[] classpath, ClassLoader parentClassLoader) + return cls.getConstructor(URL[].class, ClassLoader.class).newInstance(classpath, classLoader.getParent()); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to load container class loader class " + containerClassLoaderName, e); + } catch (NoSuchMethodException e) { + throw new RuntimeException("Container class loader must have a public constructor with " + + "parameters (URL[] classpath, ClassLoader parent)", e); + } catch (InstantiationException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException("Failed to create container class loader of class " + containerClassLoaderName, e); + } } private static void addClassPathsToList(List<URL> urls, File classpathFile) throws IOException { @@ -172,7 +197,6 @@ public final class TwillLauncher { * Populates a list of {@link File} under the given directory that has ".jar" as extension. */ private static List<File> listJarFiles(File dir, List<File> result) { - System.out.println("listing jars for " + dir.getAbsolutePath()); File[] files = dir.listFiles(); if (files == null || files.length == 0) { return result; http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java index c8abf4f..52e18eb 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java @@ -146,11 +146,12 @@ final class YarnTwillPreparer implements TwillPreparer { private final Credentials credentials; private final Map<String, Map<String, String>> logLevels = Maps.newHashMap(); private final LocationCache locationCache; + private final Map<String, Integer> maxRetries = Maps.newHashMap(); private String schedulerQueue; private String extraOptions; private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG; private ClassAcceptor classAcceptor; - private final Map<String, Integer> maxRetries = Maps.newHashMap(); + private String classLoaderClassName; YarnTwillPreparer(Configuration config, TwillSpecification twillSpec, RunId runId, String zkConnectString, Location appLocation, String extraOptions, @@ -350,6 +351,12 @@ final class YarnTwillPreparer implements TwillPreparer { } @Override + public TwillPreparer setClassLoader(String classLoaderClassName) { + this.classLoaderClassName = classLoaderClassName; + return this; + } + + @Override public TwillController start() { return start(Constants.APPLICATION_MAX_START_SECONDS, TimeUnit.SECONDS); } @@ -365,6 +372,8 @@ final class YarnTwillPreparer implements TwillPreparer { @Override public ProcessController<YarnApplicationReport> call() throws Exception { + String extraOptions = getExtraOptions(); + // Local files needed by AM Map<String, LocalFile> localFiles = Maps.newHashMap(); @@ -379,7 +388,7 @@ final class YarnTwillPreparer implements TwillPreparer { saveSpecification(twillSpec, runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC)); saveLogback(runtimeConfigDir.resolve(Constants.Files.LOGBACK_TEMPLATE)); saveClassPaths(runtimeConfigDir); - saveJvmOptions(runtimeConfigDir.resolve(Constants.Files.JVM_OPTIONS)); + saveJvmOptions(extraOptions, debugOptions, runtimeConfigDir.resolve(Constants.Files.JVM_OPTIONS)); saveArguments(new Arguments(arguments, runnableArgs), runtimeConfigDir.resolve(Constants.Files.ARGUMENTS)); saveEnvironments(runtimeConfigDir.resolve(Constants.Files.ENVIRONMENTS)); @@ -409,7 +418,7 @@ final class YarnTwillPreparer implements TwillPreparer { "-Dtwill.app=$" + Constants.TWILL_APP_NAME, "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", "-Xmx" + memory + "m", - extraOptions == null ? "" : extraOptions, + extraOptions, TwillLauncher.class.getName(), ApplicationMasterMain.class.getName(), Boolean.FALSE.toString()) @@ -452,6 +461,17 @@ final class YarnTwillPreparer implements TwillPreparer { return new File(config.get(Configs.Keys.LOCAL_STAGING_DIRECTORY, Configs.Defaults.LOCAL_STAGING_DIRECTORY)); } + /** + * Returns the extra options for the container JVM. + */ + private String getExtraOptions() { + String extraOptions = this.extraOptions == null ? "" : this.extraOptions; + if (classLoaderClassName != null) { + extraOptions += " -D" + Constants.TWILL_CONTAINER_CLASSLOADER + "=" + classLoaderClassName; + } + return extraOptions; + } + private void setEnv(String runnableName, Map<String, String> env, boolean overwrite) { Map<String, String> environment = environments.get(runnableName); if (environment == null) { @@ -742,9 +762,9 @@ final class YarnTwillPreparer implements TwillPreparer { Joiner.on(':').join(classPaths).getBytes(StandardCharsets.UTF_8)); } - private void saveJvmOptions(final Path targetPath) throws IOException { - if ((extraOptions == null || extraOptions.isEmpty()) && - JvmOptions.DebugOptions.NO_DEBUG.equals(debugOptions)) { + private void saveJvmOptions(String extraOptions, + JvmOptions.DebugOptions debugOptions, final Path targetPath) throws IOException { + if (extraOptions.isEmpty() && JvmOptions.DebugOptions.NO_DEBUG.equals(debugOptions)) { // If no vm options, no need to localize the file. return; } http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoader.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoader.java b/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoader.java new file mode 100644 index 0000000..51c1dc0 --- /dev/null +++ b/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoader.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.yarn; + +import org.apache.twill.api.ServiceAnnouncer; +import org.apache.twill.common.Cancellable; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.GeneratorAdapter; +import org.objectweb.asm.commons.Method; + +import java.net.URL; +import java.net.URLClassLoader; + +/** + * ClassLoader that generates a new class for the {@link CustomClassLoaderTestRun}. + */ +public final class CustomClassLoader extends URLClassLoader { + + public CustomClassLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + } + + @Override + protected Class<?> findClass(String name) throws ClassNotFoundException { + if (!CustomClassLoaderRunnable.GENERATED_CLASS_NAME.equals(name)) { + return super.findClass(name); + } + + // Generate a class that look like this: + // + // public class Generated { + // + // public void announce(ServiceAnnouncer announcer, String serviceName, int port) { + // announcer.announce(serviceName, port); + // } + // } + Type generatedClassType = Type.getObjectType(CustomClassLoaderRunnable.GENERATED_CLASS_NAME.replace('.', '/')); + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); + cw.visit(Opcodes.V1_7, Opcodes.ACC_PUBLIC + Opcodes.ACC_FINAL, + generatedClassType.getInternalName(), null, Type.getInternalName(Object.class), null); + + // Generate the default constructor, which just call super(); + Method constructor = new Method("<init>", Type.VOID_TYPE, new Type[0]); + GeneratorAdapter mg = new GeneratorAdapter(Opcodes.ACC_PUBLIC, constructor, null, null, cw); + mg.loadThis(); + mg.invokeConstructor(Type.getType(Object.class), constructor); + mg.returnValue(); + mg.endMethod(); + + // Generate the announce method + Method announce = new Method("announce", Type.VOID_TYPE, new Type[] { + Type.getType(ServiceAnnouncer.class), Type.getType(String.class), Type.INT_TYPE + }); + mg = new GeneratorAdapter(Opcodes.ACC_PUBLIC, announce, null, null, cw); + mg.loadArg(0); + mg.loadArg(1); + mg.loadArg(2); + mg.invokeInterface(Type.getType(ServiceAnnouncer.class), + new Method("announce", Type.getType(Cancellable.class), new Type[] { + Type.getType(String.class), Type.INT_TYPE + })); + mg.pop(); + mg.returnValue(); + mg.endMethod(); + cw.visitEnd(); + + byte[] byteCode = cw.toByteArray(); + return defineClass(CustomClassLoaderRunnable.GENERATED_CLASS_NAME, byteCode, 0, byteCode.length); + } +} http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderRunnable.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderRunnable.java b/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderRunnable.java new file mode 100644 index 0000000..66bcd42 --- /dev/null +++ b/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderRunnable.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.yarn; + +import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.twill.api.AbstractTwillRunnable; +import org.apache.twill.api.ServiceAnnouncer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CountDownLatch; + +/** + * Runnable for testing custom classloader + */ +public final class CustomClassLoaderRunnable extends AbstractTwillRunnable { + + static final String SERVICE_NAME = "custom.service"; + static final String GENERATED_CLASS_NAME = "org.apache.twill.test.Generated"; + + private static final Logger LOG = LoggerFactory.getLogger(CustomClassLoaderRunnable.class); + + private final CountDownLatch stopLatch = new CountDownLatch(1); + + @Override + public void run() { + try { + Class<?> cls = Class.forName(GENERATED_CLASS_NAME); + java.lang.reflect.Method announce = cls.getMethod("announce", ServiceAnnouncer.class, String.class, int.class); + announce.invoke(cls.newInstance(), getContext(), SERVICE_NAME, 54321); + Uninterruptibles.awaitUninterruptibly(stopLatch); + } catch (Exception e) { + LOG.error("Failed to call announce on " + GENERATED_CLASS_NAME, e); + } + } + + @Override + public void stop() { + stopLatch.countDown(); + } +} http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderTestRun.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderTestRun.java new file mode 100644 index 0000000..0ac43a6 --- /dev/null +++ b/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderTestRun.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.yarn; + +import org.apache.twill.api.TwillController; +import org.apache.twill.api.logging.PrinterLogHandler; +import org.junit.Assert; +import org.junit.Test; + +import java.io.PrintWriter; + +/** + * Unit test for testing custom classloader for containers. + */ +public class CustomClassLoaderTestRun extends BaseYarnTest { + + @Test + public void testCustomClassLoader() throws Exception { + TwillController controller = getTwillRunner().prepare(new CustomClassLoaderRunnable()) + .setClassLoader(CustomClassLoader.class.getName()) + .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) + .start(); + + Assert.assertTrue(waitForSize(controller.discoverService(CustomClassLoaderRunnable.SERVICE_NAME), 1, 120)); + controller.terminate().get(); + } +} http://git-wip-us.apache.org/repos/asf/twill/blob/c8e2a615/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java index 56172da..0911a3d 100644 --- a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java +++ b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java @@ -26,6 +26,7 @@ import org.junit.runners.Suite; @RunWith(Suite.class) @Suite.SuiteClasses({ ContainerSizeTestRun.class, + CustomClassLoaderTestRun.class, DebugTestRun.class, DistributeShellTestRun.class, EchoServerTestRun.class,
