This is an automated email from the ASF dual-hosted git repository. harishjp pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/tez.git
commit 59419dd3e3817d52f3a7ace34f122c92bea03d33 Author: Harish JP <[email protected]> AuthorDate: Thu Aug 13 11:59:03 2020 +0530 Revert "TEZ-4223 - Adding new jars or resources after the first DAG runs does not work." This reverts commit 6fc75ad6e9b1601b8b14dd85fa9b0aea53585fba. --- .../org/apache/tez/common/ReflectionUtils.java | 43 ++++++++++++++++++++- .../java/org/apache/tez/common/TezClassLoader.java | 45 ++++++++++++++-------- .../org/apache/tez/common/TestReflectionUtils.java | 4 +- .../java/org/apache/tez/dag/app/DAGAppMaster.java | 2 - 4 files changed, 73 insertions(+), 21 deletions(-) diff --git a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java index 73becda..9f7c5d3 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java @@ -19,14 +19,17 @@ package org.apache.tez.common; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; +import java.net.URLClassLoader; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.tez.dag.api.TezReflectionException; +import org.apache.tez.dag.api.TezUncheckedException; @Private public class ReflectionUtils { @@ -107,10 +110,46 @@ public class ReflectionUtils { } @Private + public static synchronized void addResourcesToClasspath(List<URL> urls) { + ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Thread + .currentThread().getContextClassLoader()); + Thread.currentThread().setContextClassLoader(classLoader); + } + + // Parameters for addResourcesToSystemClassLoader + private static final Class<?>[] parameters = new Class[]{URL.class}; + private static Method sysClassLoaderMethod = null; + + @Private public static synchronized void addResourcesToSystemClassLoader(List<URL> urls) { - TezClassLoader classLoader = TezClassLoader.getInstance(); + ClassLoader sysLoader = getSystemClassLoader(); + if (sysClassLoaderMethod == null) { + Class<?> sysClass = TezClassLoader.class; + Method method; + try { + method = sysClass.getDeclaredMethod("addURL", parameters); + } catch (SecurityException e) { + throw new TezUncheckedException("Failed to get handle on method addURL", e); + } catch (NoSuchMethodException e) { + throw new TezUncheckedException("Failed to get handle on method addURL", e); + } + method.setAccessible(true); + sysClassLoaderMethod = method; + } for (URL url : urls) { - classLoader.addURL(url); + try { + sysClassLoaderMethod.invoke(sysLoader, new Object[] { url }); + } catch (IllegalArgumentException e) { + throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e); + } catch (IllegalAccessException e) { + throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e); + } catch (InvocationTargetException e) { + throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e); + } } } + + private static ClassLoader getSystemClassLoader() { + return TezClassLoader.getInstance(); + } } diff --git a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java index 1842a19..2679efa 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java @@ -13,24 +13,30 @@ */ package org.apache.tez.common; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Arrays; -/** - * ClassLoader to allow addition of new paths to classpath in the runtime. - * - * It uses URLClassLoader with this class' classloader as parent classloader. - * And hence first delegates the resource loading to parent and then to the URLs - * added. The process must be setup to use by invoking setupTezClassLoader() which sets - * the global TezClassLoader as current thread context class loader. All threads - * created will inherit the classloader and hence will resolve the class/resource - * from TezClassLoader. - */ public class TezClassLoader extends URLClassLoader { - private static TezClassLoader INSTANCE = new TezClassLoader(); + private static TezClassLoader INSTANCE; + + static { + INSTANCE = AccessController.doPrivileged(new PrivilegedAction<TezClassLoader>() { + ClassLoader sysLoader = TezClassLoader.class.getClassLoader(); + + public TezClassLoader run() { + return new TezClassLoader( + sysLoader instanceof URLClassLoader ? ((URLClassLoader) sysLoader).getURLs() : extractClassPathEntries(), + sysLoader); + } + }); + } - private TezClassLoader() { - super(new URL[] {}, TezClassLoader.class.getClassLoader()); + public TezClassLoader(URL[] urls, ClassLoader classLoader) { + super(urls, classLoader); } public void addURL(URL url) { @@ -41,7 +47,16 @@ public class TezClassLoader extends URLClassLoader { return INSTANCE; } - public static void setupTezClassLoader() { - Thread.currentThread().setContextClassLoader(INSTANCE); + private static URL[] extractClassPathEntries() { + String pathSeparator = System.getProperty("path.separator"); + String[] classPathEntries = System.getProperty("java.class.path").split(pathSeparator); + URL[] cp = Arrays.asList(classPathEntries).stream().map(s -> { + try { + return new URL("file://" + s); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + }).toArray(URL[]::new); + return cp; } } diff --git a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java index ed3814d..2fbd35c 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java @@ -58,7 +58,7 @@ public class TestReflectionUtils { @Test(timeout = 5000) public void testAddResourceToClasspath() throws IOException, TezException { - TezClassLoader.setupTezClassLoader(); + String rsrcName = "dummyfile.xml"; FileSystem localFs = FileSystem.getLocal(new Configuration()); Path p = new Path(rsrcName); @@ -78,7 +78,7 @@ public class TestReflectionUtils { urlForm = urlForm.substring(0, urlForm.lastIndexOf('/') + 1); URL url = new URL(urlForm); - ReflectionUtils.addResourcesToSystemClassLoader(Collections.singletonList(url)); + ReflectionUtils.addResourcesToClasspath(Collections.singletonList(url)); loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index fcfb883..7e5a7a9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2328,8 +2328,6 @@ public class DAGAppMaster extends AbstractService { public static void main(String[] args) { try { - // Install the tez class loader, which can be used add new resources - TezClassLoader.setupTezClassLoader(); Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); final String pid = System.getenv().get("JVM_PID"); String containerIdStr =
