Updated Branches: refs/heads/trunk fd0143b5a -> 79f941b7e
SQOOP-796: Unable to use new Hadoop environment variables (Sean Mackrory via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/79f941b7 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/79f941b7 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/79f941b7 Branch: refs/heads/trunk Commit: 79f941b7e14df9b555084a7eee05980221d05f9e Parents: fd0143b Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Jan 10 09:01:06 2013 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Jan 10 09:01:06 2013 -0800 ---------------------------------------------------------------------- bin/configure-sqoop | 29 ++++++++++++--- bin/sqoop | 2 +- conf/sqoop-env-template.sh | 5 ++- src/docs/man/common-args.txt | 7 +++- src/docs/man/sqoop.txt | 10 ++++-- src/docs/user/common-args.txt | 2 +- src/docs/user/help.txt | 3 +- src/docs/user/tools.txt | 27 ++++++++------ src/java/org/apache/sqoop/SqoopOptions.java | 16 ++++---- .../org/apache/sqoop/orm/CompilationManager.java | 21 ++++++----- src/java/org/apache/sqoop/tool/BaseSqoopTool.java | 10 ++++-- src/scripts/run-perftest.sh | 2 +- 12 files changed, 86 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/bin/configure-sqoop ---------------------------------------------------------------------- diff --git a/bin/configure-sqoop b/bin/configure-sqoop index ee1dd7b..61ff3f2 100755 --- a/bin/configure-sqoop +++ b/bin/configure-sqoop @@ -40,17 +40,33 @@ fi # Find paths to our dependency systems. If they are unset, use CDH defaults. -if [ -z "${HADOOP_HOME}" ]; then - HADOOP_HOME=/usr/lib/hadoop +if [ -z "${HADOOP_COMMON_HOME}" ]; then + if [ -n "${HADOOP_HOME}" ]; then + HADOOP_COMMON_HOME=${HADOOP_HOME} + else + HADOOP_COMMON_HOME=/usr/lib/hadoop + fi +fi +if [ -z "${HADOOP_MAPRED_HOME}" ]; then + if [ -n "${HADOOP_HOME}" ]; then + HADOOP_MAPRED_HOME=${HADOOP_HOME} + else + HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce + fi fi if [ -z "${HBASE_HOME}" ]; then HBASE_HOME=/usr/lib/hbase fi # Check: If we can't find our dependencies, give up here. -if [ ! -d "${HADOOP_HOME}" ]; then - echo "Error: $HADOOP_HOME does not exist!" - echo 'Please set $HADOOP_HOME to the root of your Hadoop installation.' +if [ ! -d "${HADOOP_COMMON_HOME}" ]; then + echo "Error: $HADOOP_COMMON_HOME does not exist!" + echo 'Please set $HADOOP_COMMON_HOME to the root of your Hadoop installation.' + exit 1 +fi +if [ ! -d "${HADOOP_MAPRED_HOME}" ]; then + echo "Error: $HADOOP_MAPRED_HOME does not exist!" + echo 'Please set $HADOOP_MAPRED_HOME to the root of your Hadoop MapReduce installation.' exit 1 fi @@ -117,6 +133,7 @@ export SQOOP_CLASSPATH export SQOOP_CONF_DIR export SQOOP_JAR_DIR export HADOOP_CLASSPATH -export HADOOP_HOME +export HADOOP_COMMON_HOME +export HADOOP_MAPRED_HOME export HBASE_HOME http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/bin/sqoop ---------------------------------------------------------------------- diff --git a/bin/sqoop b/bin/sqoop index bc69083..0595714 100755 --- a/bin/sqoop +++ b/bin/sqoop @@ -98,4 +98,4 @@ bin=`dirname ${prgm}` bin=`cd ${bin} && pwd` source ${bin}/configure-sqoop "${bin}" -exec ${HADOOP_HOME}/bin/hadoop org.apache.sqoop.Sqoop "$@" +exec ${HADOOP_COMMON_HOME}/bin/hadoop org.apache.sqoop.Sqoop "$@" http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/conf/sqoop-env-template.sh ---------------------------------------------------------------------- diff --git a/conf/sqoop-env-template.sh b/conf/sqoop-env-template.sh index 0294f17..90bde78 100644 --- a/conf/sqoop-env-template.sh +++ b/conf/sqoop-env-template.sh @@ -20,7 +20,10 @@ # Set Hadoop-specific environment variables here. #Set path to where bin/hadoop is available -#export HADOOP_HOME= +#export HADOOP_COMMON_HOME= + +#Set path to where hadoop-*-core.jar is available +#export HADOOP_MAPRED_HOME= #set the path to where bin/hbase is available #export HBASE_HOME= http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/man/common-args.txt ---------------------------------------------------------------------- diff --git a/src/docs/man/common-args.txt b/src/docs/man/common-args.txt index 129f5d1..cf9c0c3 100644 --- a/src/docs/man/common-args.txt +++ b/src/docs/man/common-args.txt @@ -33,8 +33,8 @@ Database connection and common options --connection-param-file (filename):: Optional properties file that provides connection parameters ---hadoop-home (dir):: - Override $HADOOP_HOME +--hadoop-mapred-home (dir):: + Override $HADOOP_MAPRED_HOME --help:: Print usage instructions @@ -51,3 +51,6 @@ Database connection and common options --verbose:: Print more information while working + +--hadoop-home (dir):: + Deprecated. Override $HADOOP_HOME http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/man/sqoop.txt ---------------------------------------------------------------------- diff --git a/src/docs/man/sqoop.txt b/src/docs/man/sqoop.txt index 44b4f4e..febe827 100644 --- a/src/docs/man/sqoop.txt +++ b/src/docs/man/sqoop.txt @@ -43,9 +43,13 @@ JAVA_HOME:: e.g., +/usr/java/default+. Hadoop (and Sqoop) requires Sun Java 1.6 which can be downloaded from http://java.sun.com. -HADOOP_HOME:: - The location of the Hadoop jar files. If you installed Hadoop via RPM - or DEB, these are in +/usr/lib/hadoop-20+. +HADOOP_COMMON_HOME:: + The location of the Hadoop Common files (specifically the bin/hadoop executable). + If you installed Hadoop via RPM or DEB, this is in +/usr/lib/hadoop+. + +HADOOP_MAPRED_HOME:: + The location of the Hadoop MapReduce files (specifically the JAR files). + If you installed Hadoop via RPM or DEB, this is in +/usr/lib/hadoop-mapreduce+. HIVE_HOME:: If you are performing a Hive import, you must identify the location of http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/user/common-args.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/common-args.txt b/src/docs/user/common-args.txt index 7c69886..0554f81 100644 --- a/src/docs/user/common-args.txt +++ b/src/docs/user/common-args.txt @@ -27,7 +27,7 @@ Argument Description use +\--driver <class-name>+ Manually specify JDBC driver class\ to use -+\--hadoop-home <dir>+ Override $HADOOP_HOME ++\--hadoop-mapred-home <dir>+ Override $HADOOP_MAPRED_HOME +\--help+ Print usage instructions +-P+ Read password from console +\--password <password>+ Set authentication password http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/user/help.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/help.txt b/src/docs/user/help.txt index 11b4f0c..24fbddc 100644 --- a/src/docs/user/help.txt +++ b/src/docs/user/help.txt @@ -68,12 +68,13 @@ Common arguments: --connect <jdbc-uri> Specify JDBC connect string --connection-manager <class-name> Specify connection manager class to use --driver <class-name> Manually specify JDBC driver class to use - --hadoop-home <dir> Override $HADOOP_HOME + --hadoop-mapred-home <dir> Override $HADOOP_MAPRED_HOME --help Print usage instructions -P Read password from console --password <password> Set authentication password --username <username> Set authentication username --verbose Print more information while working + --hadoop-home <dir> Deprecated. Override $HADOOP_HOME Import control arguments: --as-avrodatafile Imports data to Avro Data Files http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/user/tools.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/tools.txt b/src/docs/user/tools.txt index ce83a24..96bf777 100644 --- a/src/docs/user/tools.txt +++ b/src/docs/user/tools.txt @@ -25,10 +25,9 @@ tool you want to use and the arguments that control the tool. If Sqoop is compiled from its own source, you can run Sqoop without a formal installation process by running the +bin/sqoop+ program. Users -of a packaged deployment of Sqoop (such as an RPM shipped with Cloudera's -Distribution for Hadoop) will see this program installed as +/usr/bin/sqoop+. -The remainder of this documentation will refer to this program as -+sqoop+. For example: +of a packaged deployment of Sqoop (such as an RPM shipped with Apache Bigtop) +will see this program installed as +/usr/bin/sqoop+. The remainder of this +documentation will refer to this program as +sqoop+. For example: ---- $ sqoop tool-name [tool-arguments] @@ -87,24 +86,29 @@ You invoke Sqoop through the program launch capability provided by Hadoop. The +sqoop+ command-line program is a wrapper which runs the +bin/hadoop+ script shipped with Hadoop. If you have multiple installations of Hadoop present on your machine, you can select the -Hadoop installation by setting the +$HADOOP_HOME+ environment -variable. +Hadoop installation by setting the +$HADOOP_COMMON_HOME+ and ++$HADOOP_MAPRED_HOME+ environment variables. For example: ---- -$ HADOOP_HOME=/path/to/some/hadoop sqoop import --arguments... +$ HADOOP_COMMON_HOME=/path/to/some/hadoop \ + HADOOP_MAPRED_HOME=/path/to/some/hadoop-mapreduce \ + sqoop import --arguments... ---- or: ---- -$ export HADOOP_HOME=/some/path/to/hadoop +$ export HADOOP_COMMON_HOME=/some/path/to/hadoop +$ export HADOOP_MAPRED_HOME=/some/path/to/hadoop-mapreduce $ sqoop import --arguments... ----- -If +$HADOOP_HOME+ is not set, Sqoop will use the default installation -location for Cloudera's Distribution for Hadoop, +/usr/lib/hadoop+. +If either of these variables are not set, Sqoop will fall back to ++$HADOOP_HOME+. If it is not set either, Sqoop will use the default +installation locations for Apache Bigtop, +/usr/lib/hadoop+ and ++/usr/lib/hadoop-mapreduce+, respectively. The active Hadoop configuration is loaded from +$HADOOP_HOME/conf/+, unless the +$HADOOP_CONF_DIR+ environment variable is set. @@ -126,12 +130,13 @@ Common arguments: --connect <jdbc-uri> Specify JDBC connect string --connect-manager <jdbc-uri> Specify connection manager class to use --driver <class-name> Manually specify JDBC driver class to use - --hadoop-home <dir> Override $HADOOP_HOME + --hadoop-mapred-home <dir>+ Override $HADOOP_MAPRED_HOME --help Print usage instructions -P Read password from console --password <password> Set authentication password --username <username> Set authentication username --verbose Print more information while working + --hadoop-home <dir>+ Deprecated. Override $HADOOP_HOME [...] http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 3e0ec3e..b0fdfa0 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -119,7 +119,7 @@ public class SqoopOptions implements Cloneable { // explicitly set by the user (false). If the former, disregard any value // for jarOutputDir saved in the metastore. @StoredAsProperty("codegen.auto.compile.dir") private boolean jarDirIsAuto; - private String hadoopHome; // not serialized to metastore. + private String hadoopMapRedHome; // not serialized to metastore. @StoredAsProperty("db.split.column") private String splitByCol; @StoredAsProperty("db.where.clause") private String whereClause; @StoredAsProperty("db.query") private String sqlQuery; @@ -775,7 +775,7 @@ public class SqoopOptions implements Cloneable { private void initDefaults(Configuration baseConfiguration) { // first, set the true defaults if nothing else happens. // default action is to run the full pipeline. - this.hadoopHome = System.getenv("HADOOP_HOME"); + this.hadoopMapRedHome = System.getenv("HADOOP_MAPRED_HOME"); this.hiveHome = getHiveHomeDefault(); @@ -1253,15 +1253,15 @@ public class SqoopOptions implements Cloneable { } /** - * Return the value of $HADOOP_HOME. - * @return $HADOOP_HOME, or null if it's not set. + * Return the value of $HADOOP_MAPRED_HOME. + * @return $HADOOP_MAPRED_HOME, or null if it's not set. */ - public String getHadoopHome() { - return hadoopHome; + public String getHadoopMapRedHome() { + return hadoopMapRedHome; } - public void setHadoopHome(String home) { - this.hadoopHome = home; + public void setHadoopMapRedHome(String home) { + this.hadoopMapRedHome = home; } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/java/org/apache/sqoop/orm/CompilationManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java index 26adcbf..92effb5 100644 --- a/src/java/org/apache/sqoop/orm/CompilationManager.java +++ b/src/java/org/apache/sqoop/orm/CompilationManager.java @@ -71,28 +71,29 @@ public class CompilationManager { } /** - * locate the hadoop-*-core.jar in $HADOOP_HOME or --hadoop-home. + * locate the hadoop-*-core.jar in $HADOOP_MAPRED_HOME or + * --hadoop-mapred-home. * If that doesn't work, check our classpath. * @return the filename of the hadoop-*-core.jar file. */ private String findHadoopCoreJar() { - String hadoopHome = options.getHadoopHome(); + String hadoopMapRedHome = options.getHadoopMapRedHome(); - if (null == hadoopHome) { - LOG.info("$HADOOP_HOME is not set"); + if (null == hadoopMapRedHome) { + LOG.info("$HADOOP_MAPRED_HOME is not set"); return Jars.getJarPathForClass(JobConf.class); } - if (!hadoopHome.endsWith(File.separator)) { - hadoopHome = hadoopHome + File.separator; + if (!hadoopMapRedHome.endsWith(File.separator)) { + hadoopMapRedHome = hadoopMapRedHome + File.separator; } - File hadoopHomeFile = new File(hadoopHome); - LOG.info("HADOOP_HOME is " + hadoopHomeFile.getAbsolutePath()); - File [] entries = hadoopHomeFile.listFiles(); + File hadoopMapRedHomeFile = new File(hadoopMapRedHome); + LOG.info("HADOOP_MAPRED_HOME is " + hadoopMapRedHomeFile.getAbsolutePath()); + File [] entries = hadoopMapRedHomeFile.listFiles(); if (null == entries) { - LOG.warn("HADOOP_HOME appears empty or missing"); + LOG.warn("HADOOP_MAPRED_HOME appears empty or missing"); return Jars.getJarPathForClass(JobConf.class); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index ea8a3f4..b4b2213 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -80,6 +80,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String SPLIT_BY_ARG = "split-by"; public static final String WHERE_ARG = "where"; public static final String HADOOP_HOME_ARG = "hadoop-home"; + public static final String HADOOP_MAPRED_HOME_ARG = "hadoop-mapred-home"; public static final String HIVE_HOME_ARG = "hive-home"; public static final String WAREHOUSE_DIR_ARG = "warehouse-dir"; public static final String TARGET_DIR_ARG = "target-dir"; @@ -385,8 +386,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { .create(PASSWORD_PROMPT_ARG)); commonOpts.addOption(OptionBuilder.withArgName("dir") - .hasArg().withDescription("Override $HADOOP_HOME") - .withLongOpt(HADOOP_HOME_ARG) + .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG") + .withLongOpt(HADOOP_MAPRED_HOME_ARG) .create()); // misc (common) @@ -750,7 +751,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { } if (in.hasOption(HADOOP_HOME_ARG)) { - out.setHadoopHome(in.getOptionValue(HADOOP_HOME_ARG)); + out.setHadoopMapRedHome(in.getOptionValue(HADOOP_HOME_ARG)); + } + if (in.hasOption(HADOOP_MAPRED_HOME_ARG)) { + out.setHadoopMapRedHome(in.getOptionValue(HADOOP_MAPRED_HOME_ARG)); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/scripts/run-perftest.sh ---------------------------------------------------------------------- diff --git a/src/scripts/run-perftest.sh b/src/scripts/run-perftest.sh index 2a4ba21..55beaa7 100755 --- a/src/scripts/run-perftest.sh +++ b/src/scripts/run-perftest.sh @@ -37,5 +37,5 @@ source ${SQOOP_HOME}/bin/configure-sqoop "${bin}" PERFTEST_CLASSES=${SQOOP_HOME}/build/perftest/classes export HADOOP_CLASSPATH=${PERFTEST_CLASSES}:${SQOOP_JAR}:${HADOOP_CLASSPATH} -${HADOOP_HOME}/bin/hadoop "$@" +${HADOOP_COMMON_HOME}/bin/hadoop "$@"
