Author: tomwhite
Date: Fri Jan 4 11:00:55 2013
New Revision: 1428781
URL: http://svn.apache.org/viewvc?rev=1428781&view=rev
Log:
YARN-286. Add a YARN ApplicationClassLoader.
Added:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
(with props)
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
(with props)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1428781&r1=1428780&r2=1428781&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri Jan 4 11:00:55 2013
@@ -30,6 +30,8 @@ Release 2.0.3-alpha - Unreleased
YARN-103. Add a yarn AM-RM client module. (Bikas Saha via sseth)
+ YARN-286. Add a YARN ApplicationClassLoader. (tomwhite)
+
IMPROVEMENTS
YARN-223. Update process tree instead of getting new process trees.
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1428781&r1=1428780&r2=1428781&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
Fri Jan 4 11:00:55 2013
@@ -143,6 +143,11 @@ public interface ApplicationConstants {
CLASSPATH("CLASSPATH"),
/**
+ * $APP_CLASSPATH
+ */
+ APP_CLASSPATH("APP_CLASSPATH"),
+
+ /**
* $LD_LIBRARY_PATH
*/
LD_LIBRARY_PATH("LD_LIBRARY_PATH"),
Added:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1428781&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
(added)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
Fri Jan 4 11:00:55 2013
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * A {@link URLClassLoader} for YARN application isolation. Classes from
+ * the application JARs are loaded in preference to the parent loader.
+ */
+@Public
+@Unstable
+public class ApplicationClassLoader extends URLClassLoader {
+
+ private static final Log LOG =
+ LogFactory.getLog(ApplicationClassLoader.class.getName());
+
+ private static final FilenameFilter JAR_FILENAME_FILTER =
+ new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".jar") || name.endsWith(".JAR");
+ }
+ };
+
+ private ClassLoader parent;
+ private List<String> systemClasses;
+
+ public ApplicationClassLoader(URL[] urls, ClassLoader parent,
+ List<String> systemClasses) {
+ super(urls, parent);
+ this.parent = parent;
+ if (parent == null) {
+ throw new IllegalArgumentException("No parent classloader!");
+ }
+ this.systemClasses = systemClasses;
+ }
+
+ public ApplicationClassLoader(String classpath, ClassLoader parent,
+ List<String> systemClasses) throws MalformedURLException {
+ this(constructUrlsFromClasspath(classpath), parent, systemClasses);
+ }
+
+ @VisibleForTesting
+ static URL[] constructUrlsFromClasspath(String classpath)
+ throws MalformedURLException {
+ List<URL> urls = new ArrayList<URL>();
+ for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
+ if (element.endsWith("/*")) {
+ String dir = element.substring(0, element.length() - 1);
+ File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
+ if (files != null) {
+ for (File file : files) {
+ urls.add(file.toURI().toURL());
+ }
+ }
+ } else {
+ File file = new File(element);
+ if (file.exists()) {
+ urls.add(new File(element).toURI().toURL());
+ }
+ }
+ }
+ return urls.toArray(new URL[urls.size()]);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ URL url = null;
+
+ if (!isSystemClass(name, systemClasses)) {
+ url= findResource(name);
+ if (url == null && name.startsWith("/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remove leading / off " + name);
+ }
+ url= findResource(name.substring(1));
+ }
+ }
+
+ if (url == null) {
+ url= parent.getResource(name);
+ }
+
+ if (url != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getResource("+name+")=" + url);
+ }
+ }
+
+ return url;
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ return this.loadClass(name, false);
+ }
+
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve)
+ throws ClassNotFoundException {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading class: " + name);
+ }
+
+ Class<?> c = findLoadedClass(name);
+ ClassNotFoundException ex = null;
+
+ if (c == null && !isSystemClass(name, systemClasses)) {
+ // Try to load class from this classloader's URLs. Note that this is like
+ // the servlet spec, not the usual Java 2 behaviour where we ask the
+ // parent to attempt to load first.
+ try {
+ c = findClass(name);
+ if (LOG.isDebugEnabled() && c != null) {
+ LOG.debug("Loaded class: " + name + " ");
+ }
+ } catch (ClassNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(e);
+ }
+ ex = e;
+ }
+ }
+
+ if (c == null) { // try parent
+ c = parent.loadClass(name);
+ if (LOG.isDebugEnabled() && c != null) {
+ LOG.debug("Loaded class from parent: " + name + " ");
+ }
+ }
+
+ if (c == null) {
+ throw ex != null ? ex : new ClassNotFoundException(name);
+ }
+
+ if (resolve) {
+ resolveClass(c);
+ }
+
+ return c;
+ }
+
+ @VisibleForTesting
+ static boolean isSystemClass(String name, List<String> systemClasses) {
+ if (systemClasses != null) {
+ String canonicalName = name.replace('/', '.');
+ while (canonicalName.startsWith(".")) {
+ canonicalName=canonicalName.substring(1);
+ }
+ for (String c : systemClasses) {
+ boolean result = true;
+ if (c.startsWith("-")) {
+ c = c.substring(1);
+ result = false;
+ }
+ if (c.endsWith(".") && canonicalName.startsWith(c)) {
+ return result;
+ } else if (canonicalName.equals(c)) {
+ return result;
+ }
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file
Propchange:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java?rev=1428781&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
(added)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
Fri Jan 4 11:00:55 2013
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static
org.apache.hadoop.yarn.util.ApplicationClassLoader.constructUrlsFromClasspath;
+import static org.apache.hadoop.yarn.util.ApplicationClassLoader.isSystemClass;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.List;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestApplicationClassLoader {
+
+ private static File testDir = new File(System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")), "appclassloader");
+
+ @Before
+ public void setUp() {
+ FileUtil.fullyDelete(testDir);
+ testDir.mkdirs();
+ }
+
+ @Test
+ public void testConstructUrlsFromClasspath() throws Exception {
+ File file = new File(testDir, "file");
+ assertTrue("Create file", file.createNewFile());
+
+ File dir = new File(testDir, "dir");
+ assertTrue("Make dir", dir.mkdir());
+
+ File jarsDir = new File(testDir, "jarsdir");
+ assertTrue("Make jarsDir", jarsDir.mkdir());
+ File nonJarFile = new File(jarsDir, "nonjar");
+ assertTrue("Create non-jar file", nonJarFile.createNewFile());
+ File jarFile = new File(jarsDir, "a.jar");
+ assertTrue("Create jar file", jarFile.createNewFile());
+
+ File nofile = new File(testDir, "nofile");
+ // don't create nofile
+
+ StringBuilder cp = new StringBuilder();
+ cp.append(file.getAbsolutePath()).append(File.pathSeparator)
+ .append(dir.getAbsolutePath()).append(File.pathSeparator)
+ .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
+ .append(nofile.getAbsolutePath()).append(File.pathSeparator)
+ .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
+
+ URL[] urls = constructUrlsFromClasspath(cp.toString());
+
+ assertEquals(3, urls.length);
+ assertEquals(file.toURI().toURL(), urls[0]);
+ assertEquals(dir.toURI().toURL(), urls[1]);
+ assertEquals(jarFile.toURI().toURL(), urls[2]);
+ // nofile should be ignored
+ }
+
+ @Test
+ public void testIsSystemClass() {
+ assertFalse(isSystemClass("org.example.Foo", null));
+ assertTrue(isSystemClass("org.example.Foo", classes("org.example.Foo")));
+ assertTrue(isSystemClass("/org.example.Foo", classes("org.example.Foo")));
+ assertTrue(isSystemClass("org.example.Foo", classes("org.example.")));
+ assertTrue(isSystemClass("net.example.Foo",
+ classes("org.example.,net.example.")));
+ assertFalse(isSystemClass("org.example.Foo",
+ classes("-org.example.Foo,org.example.")));
+ assertTrue(isSystemClass("org.example.Bar",
+ classes("-org.example.Foo.,org.example.")));
+ }
+
+ private List<String> classes(String classes) {
+ return Lists.newArrayList(Splitter.on(',').split(classes));
+ }
+
+ @Test
+ public void testGetResource() throws IOException {
+ URL testJar = makeTestJar().toURI().toURL();
+
+ ClassLoader currentClassLoader = getClass().getClassLoader();
+ ClassLoader appClassloader = new ApplicationClassLoader(
+ new URL[] { testJar }, currentClassLoader, null);
+
+ assertNull("Resource should be null for current classloader",
+ currentClassLoader.getResourceAsStream("resource.txt"));
+
+ InputStream in = appClassloader.getResourceAsStream("resource.txt");
+ assertNotNull("Resource should not be null for app classloader", in);
+ assertEquals("hello", IOUtils.toString(in));
+ }
+
+ private File makeTestJar() throws IOException {
+ File jarFile = new File(testDir, "test.jar");
+ JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile));
+ ZipEntry entry = new ZipEntry("resource.txt");
+ out.putNextEntry(entry);
+ out.write("hello".getBytes());
+ out.closeEntry();
+ out.close();
+ return jarFile;
+ }
+
+}
Propchange:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
------------------------------------------------------------------------------
svn:eol-style = native
