HIVE-10858 WebHCat specific resources should be added to HADOOP_CLASSPATH first(Eugene Koifman, Reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fa2dd839 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fa2dd839 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fa2dd839 Branch: refs/heads/llap Commit: fa2dd83913b2464d81504a97a9244187889a3c9f Parents: 7ad9b14 Author: Eugene Koifman <[email protected]> Authored: Fri May 29 15:28:21 2015 -0700 Committer: Eugene Koifman <[email protected]> Committed: Fri May 29 15:28:21 2015 -0700 ---------------------------------------------------------------------- .../templeton/tool/JobSubmissionConstants.java | 1 + .../hcatalog/templeton/tool/LaunchMapper.java | 27 +++++++++++++------- 2 files changed, 19 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/fa2dd839/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java index 1d560b6..d3dc3f7 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java @@ -55,6 +55,7 @@ public interface JobSubmissionConstants { public static final String MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER = "__MR_JOB_TAGS_OPTION=MR_JOB_TAGS_JOBID__"; + public static final String HADOOP_CLASSPATH = "HADOOP_CLASSPATH"; /** * constants needed for Pig job submission * The string values here are what Pig expects to see in it's environment http://git-wip-us.apache.org/repos/asf/hive/blob/fa2dd839/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index 3edd449..91fe247 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -107,12 +107,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> } jdbcJars.setLength(jdbcJars.length() - 1); //this makes the jars available to Sqoop client - if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) { - env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + jdbcJars.toString()); - } - else { - env.put("HADOOP_CLASSPATH", jdbcJars.toString()); - } + prependPathToVariable(HADOOP_CLASSPATH, env, jdbcJars.toString()); } } private static void handleHadoopClasspathExtras(Configuration conf, Map<String, String> env) @@ -134,11 +129,25 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> paths.append(File.pathSeparator); } paths.setLength(paths.length() - 1); - if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) { - env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + paths); + prependPathToVariable(HADOOP_CLASSPATH, env, paths.toString()); + } + /** + * Ensures that {@code paths} are prepended to {@code pathVarName} and made available to forked child + * process. + * @param paths properly separated list of paths + */ + private static void prependPathToVariable(String pathVarName, Map<String, String> env, String paths) { + if(!TempletonUtils.isset(pathVarName) || !TempletonUtils.isset(paths) || env == null) { + return; + } + if(TempletonUtils.isset(env.get(pathVarName))) { + env.put(pathVarName, paths + File.pathSeparator + env.get(pathVarName)); + } + else if(TempletonUtils.isset(System.getenv(pathVarName))) { + env.put(pathVarName, paths + File.pathSeparator + System.getenv(pathVarName)); } else { - env.put("HADOOP_CLASSPATH", paths.toString()); + env.put(pathVarName, paths); } } protected Process startJob(Context context, String user, String overrideClasspath)
