Repository: hive Updated Branches: refs/heads/master 8c546fe33 -> eeaea7f84
HIVE-14229: the jars in hive.aux.jar.paths are not added to session classpath (Aihua Xu, reviewed by Mohit Sabharwal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eeaea7f8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eeaea7f8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eeaea7f8 Branch: refs/heads/master Commit: eeaea7f841429c3e785e485b1887f3f7157bd6ed Parents: 8c546fe Author: Aihua Xu <aihu...@apache.org> Authored: Mon Jul 18 15:40:18 2016 -0400 Committer: Aihua Xu <aihu...@apache.org> Committed: Thu Jul 21 11:17:34 2016 -0400 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/Utilities.java | 4 +- .../hive/ql/processors/ReloadProcessor.java | 2 +- .../hadoop/hive/ql/session/SessionState.java | 65 +++++++++++--------- .../hive/ql/session/TestSessionState.java | 8 +-- .../service/cli/session/HiveSessionImpl.java | 3 +- 5 files changed, 45 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/eeaea7f8/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index a376023..cadda8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -1760,7 +1760,7 @@ public final class Utilities { * @param newPaths * Array of classpath elements */ - public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception { + public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) { final URLClassLoader loader = (URLClassLoader) cloader; if (useExistingClassLoader(cloader)) { final UDFClassLoader udfClassLoader = (UDFClassLoader) loader; @@ -1791,7 +1791,7 @@ public final class Utilities { * @param pathsToRemove * Array of classpath elements */ - public static void removeFromClassPath(String[] pathsToRemove) throws Exception { + public static void removeFromClassPath(String[] pathsToRemove) throws IOException { Thread curThread = Thread.currentThread(); URLClassLoader loader = (URLClassLoader) curThread.getContextClassLoader(); Set<URL> newPath = new HashSet<URL>(Arrays.asList(loader.getURLs())); http://git-wip-us.apache.org/repos/asf/hive/blob/eeaea7f8/ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java index 7a59833..05b7fa8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java @@ -39,7 +39,7 @@ public class ReloadProcessor implements CommandProcessor{ public CommandProcessorResponse run(String command) throws CommandNeedRetryException { SessionState ss = SessionState.get(); try { - ss.reloadAuxJars(); + ss.loadReloadableAuxJars(); } catch (IOException e) { LOG.error("fail to reload auxiliary jar files", e); return CommandProcessorResponse.create(e); http://git-wip-us.apache.org/repos/asf/hive/blob/eeaea7f8/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 03010ea..83a8610 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.lang.management.ManagementFactory; -import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URLClassLoader; @@ -44,7 +43,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -76,7 +75,6 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; import org.apache.hadoop.hive.ql.security.authorization.plugin.AuthorizationMetaStoreFilterHook; @@ -1099,8 +1097,28 @@ public class SessionState { } } - // reloading the jars under the path specified in hive.reloadable.aux.jars.path property - public void reloadAuxJars() throws IOException { + /** + * Load the jars under the path specified in hive.aux.jars.path property. Add + * the jars to the classpath so the local task can refer to them. + * @throws IOException + */ + public void loadAuxJars() throws IOException { + String[] jarPaths = StringUtils.split(sessionConf.getAuxJars(), ','); + if (ArrayUtils.isEmpty(jarPaths)) return; + + URLClassLoader currentCLoader = + (URLClassLoader) SessionState.get().getConf().getClassLoader(); + currentCLoader = + (URLClassLoader) Utilities.addToClassPath(currentCLoader, jarPaths); + sessionConf.setClassLoader(currentCLoader); + Thread.currentThread().setContextClassLoader(currentCLoader); + } + + /** + * Reload the jars under the path specified in hive.reloadable.aux.jars.path property. + * @throws IOException + */ + public void loadReloadableAuxJars() throws IOException { final Set<String> reloadedAuxJars = new HashSet<String>(); final String renewableJarPath = sessionConf.getVar(ConfVars.HIVERELOADABLEJARS); @@ -1117,32 +1135,21 @@ public class SessionState { } // remove the previous renewable jars - try { - if (preReloadableAuxJars != null && !preReloadableAuxJars.isEmpty()) { - Utilities.removeFromClassPath(preReloadableAuxJars.toArray(new String[0])); - } - } catch (Exception e) { - String msg = "Fail to remove the reloaded jars loaded last time: " + e; - throw new IOException(msg, e); + if (preReloadableAuxJars != null && !preReloadableAuxJars.isEmpty()) { + Utilities.removeFromClassPath(preReloadableAuxJars.toArray(new String[0])); } - try { - if (reloadedAuxJars != null && !reloadedAuxJars.isEmpty()) { - URLClassLoader currentCLoader = - (URLClassLoader) SessionState.get().getConf().getClassLoader(); - currentCLoader = - (URLClassLoader) Utilities.addToClassPath(currentCLoader, - reloadedAuxJars.toArray(new String[0])); - sessionConf.setClassLoader(currentCLoader); - Thread.currentThread().setContextClassLoader(currentCLoader); - } - preReloadableAuxJars.clear(); - preReloadableAuxJars.addAll(reloadedAuxJars); - } catch (Exception e) { - String msg = - "Fail to add jars from the path specified in hive.reloadable.aux.jars.path property: " + e; - throw new IOException(msg, e); + if (reloadedAuxJars != null && !reloadedAuxJars.isEmpty()) { + URLClassLoader currentCLoader = + (URLClassLoader) SessionState.get().getConf().getClassLoader(); + currentCLoader = + (URLClassLoader) Utilities.addToClassPath(currentCLoader, + reloadedAuxJars.toArray(new String[0])); + sessionConf.setClassLoader(currentCLoader); + Thread.currentThread().setContextClassLoader(currentCLoader); } + preReloadableAuxJars.clear(); + preReloadableAuxJars.addAll(reloadedAuxJars); } static void registerJars(List<String> newJars) throws IllegalArgumentException { @@ -1165,7 +1172,7 @@ public class SessionState { Utilities.removeFromClassPath(jarsToUnregister.toArray(new String[0])); console.printInfo("Deleted " + jarsToUnregister + " from class path"); return true; - } catch (Exception e) { + } catch (IOException e) { console.printError("Unable to unregister " + jarsToUnregister + "\nException: " + e.getMessage(), "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); http://git-wip-us.apache.org/repos/asf/hive/blob/eeaea7f8/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java index 145a531..fdebb94 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java @@ -202,7 +202,7 @@ public class TestSessionState { try { dist = new File(reloadFolder.getAbsolutePath() + File.separator + reloadClazzFileName); Files.copy(new File(HiveTestUtils.getFileFromClasspath(clazzDistFileName)), dist); - ss.reloadAuxJars(); + ss.loadReloadableAuxJars(); Assert.assertEquals("version1", getReloadedClazzVersion(ss.getConf().getClassLoader())); } catch (Exception e) { LOG.error("Reload auxiliary jar test fail with message: ", e); @@ -234,7 +234,7 @@ public class TestSessionState { dist = new File(reloadFolder.getAbsolutePath() + File.separator + reloadClazzFileName); Files.copy(new File(HiveTestUtils.getFileFromClasspath(clazzDistFileName)), dist); - ss.reloadAuxJars(); + ss.loadReloadableAuxJars(); Assert.assertEquals("version1", getReloadedClazzVersion(ss.getConf().getClassLoader())); @@ -242,11 +242,11 @@ public class TestSessionState { FileUtils.deleteQuietly(dist); Files.copy(new File(HiveTestUtils.getFileFromClasspath(clazzV2FileName)), dist); - ss.reloadAuxJars(); + ss.loadReloadableAuxJars(); Assert.assertEquals("version2", getReloadedClazzVersion(ss.getConf().getClassLoader())); FileUtils.deleteQuietly(dist); - ss.reloadAuxJars(); + ss.loadReloadableAuxJars(); } catch (Exception e) { LOG.error("refresh existing jar file case failed with message: ", e); Assert.fail(e.getMessage()); http://git-wip-us.apache.org/repos/asf/hive/blob/eeaea7f8/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index bb53bc6..ef37ff5 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -168,7 +168,8 @@ public class HiveSessionImpl implements HiveSession { sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses()); SessionState.start(sessionState); try { - sessionState.reloadAuxJars(); + sessionState.loadAuxJars(); + sessionState.loadReloadableAuxJars(); } catch (IOException e) { String msg = "Failed to load reloadable jar file path: " + e; LOG.error(msg, e);
