Updated Branches: refs/heads/trunk 28f415ff2 -> c688915e8
SQOOP-1259: Sqoop on Windows can't run HCatalog/HBase multinode jobs (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c688915e Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c688915e Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c688915e Branch: refs/heads/trunk Commit: c688915e8e88f48563181f5c47826e1a026c2374 Parents: 28f415f Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Dec 16 03:18:59 2013 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Dec 16 03:18:59 2013 -0800 ---------------------------------------------------------------------- .gitattributes | 1 + bin/configure-sqoop.cmd | 33 ++++++++++++++++++-- .../mapreduce/hcat/SqoopHCatUtilities.java | 6 +++- 3 files changed, 37 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/c688915e/.gitattributes ---------------------------------------------------------------------- diff --git a/.gitattributes b/.gitattributes index bd835f9..64ed28f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -31,5 +31,6 @@ *.sh text eol=lf *.bat text eol=crlf +*.cmd text eol=crlf *.csproj text merge=union eol=crlf *.sln text merge=union eol=crlf http://git-wip-us.apache.org/repos/asf/sqoop/blob/c688915e/bin/configure-sqoop.cmd ---------------------------------------------------------------------- diff --git a/bin/configure-sqoop.cmd b/bin/configure-sqoop.cmd index 4598bc8..e12a5be 100644 --- a/bin/configure-sqoop.cmd +++ b/bin/configure-sqoop.cmd @@ -72,6 +72,20 @@ if not defined HBASE_HOME ( echo Warning: HBASE_HOME and HBASE_VERSION not set. ) ) +:: Check for HCatalog dependency +if not defined HCAT_HOME ( + if defined HCATALOG_HOME ( + set HCAT_HOME=%HCATALOG_HOME% + ) else ( + echo Warning: HCAT_HOME not set + ) +) + +if not exist "%HCATALOG_HOME%" ( + echo Warning: HCATALOG_HOME does not exist! HCatalog imports will fail. + echo Please set HCATALOG_HOME to the root of your HCatalog installation. +) + :: :: Check for Accumulo dependency if not defined ACCUMULO_HOME ( @@ -125,11 +139,25 @@ if exist "%SQOOP_HOME%\lib" ( call :add_dir_to_classpath %SQOOP_HOME%\lib ) +:: Add HCatalog Home to the dependency list so that newer thrift libraries are +:: used instead of HBase version + +if exist "%HCATALOG_HOME%" ( + if defined PYTHON_CMD ( + for /F "usebackq eol==" %%G IN ( `%PYTHON_CMD% %HCAT_HOME%\bin\hcat.py -classpath`) DO SET SQOOP_CLASSPATH=%SQOOP_CLASSPATH%;%%G + ) else ( + for /f "usebackq eol==" %%G IN ( `python %HCAT_HOME%\bin\hcat.py -classpath`) DO SET SQOOP_CLASSPATH=%SQOOP_CLASSPATH%;%%G + ) +) + + :: Add HBase to dependency list if exist "%HBASE_HOME%" ( call :add_dir_to_classpath %HBASE_HOME% call :add_dir_to_classpath %HBASE_HOME%\lib -) + set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%HBASE_HOME%\conf + ) + :: :: Add Accumulo to dependency list if exist "%ACCUMULO_HOME%" ( @@ -147,7 +175,7 @@ if not defined ZOOCFGDIR ( ) if "%ZOOCFGDIR%" NEQ "" ( - call :add_dir_to_classpath %ZOOCFGDIR% + set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%ZOOCFGDIR% ) call :add_dir_to_classpath %SQOOP_CONF_DIR% @@ -173,3 +201,4 @@ if not "%1"=="" ( set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%1\* ) goto :eof + http://git-wip-us.apache.org/repos/asf/sqoop/blob/c688915e/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java index 09652df..25a39be 100644 --- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java +++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java @@ -105,7 +105,7 @@ public final class SqoopHCatUtilities { "sqoop.hcat.debug.import.mapper"; public static final String DEBUG_HCAT_EXPORT_MAPPER_PROP = "sqoop.hcat.debug.export.mapper"; - private static final String HCATCMD = Shell.WINDOWS ? "hcat.cmd" : "hcat"; + private static final String HCATCMD = Shell.WINDOWS ? "hcat.py" : "hcat"; private SqoopOptions options; private ConnManager connManager; private String hCatTableName; @@ -970,6 +970,10 @@ public final class SqoopHCatUtilities { // run HCat command with the given args String hCatProgram = getHCatPath(); ArrayList<String> args = new ArrayList<String>(); + if (Shell.WINDOWS) { + // windows depends on python to be available + args.add("python"); + } args.add(hCatProgram); if (cmdLine != null && cmdLine.length > 0) { for (String s : cmdLine) {
