Updated Branches: refs/heads/trunk 8b3742ad6 -> f5006b600
SQOOP-955: Fix Sqoop unit test failures on Windows (Ahmed El Baz via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f5006b60 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f5006b60 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f5006b60 Branch: refs/heads/trunk Commit: f5006b60059dda44e6b77e6f29749e2fbcab5bf1 Parents: 8b3742a Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Apr 15 09:56:10 2013 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Apr 15 09:57:25 2013 -0700 ---------------------------------------------------------------------- src/java/org/apache/sqoop/hive/HiveImport.java | 8 +- .../org/apache/sqoop/orm/CompilationManager.java | 6 + .../org/apache/sqoop/util/ClassLoaderStack.java | 2 +- .../com/cloudera/sqoop/hive/TestHiveImport.java | 7 +- src/test/com/cloudera/sqoop/io/TestNamedFifo.java | 8 + src/test/com/cloudera/sqoop/lib/TestBlobRef.java | 2 +- src/test/com/cloudera/sqoop/lib/TestClobRef.java | 2 +- .../com/cloudera/sqoop/orm/TestClassWriter.java | 6 + testdata/hive/bin/hive.cmd | 112 +++++++++++++++ 9 files changed, 144 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/java/org/apache/sqoop/hive/HiveImport.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/hive/HiveImport.java b/src/java/org/apache/sqoop/hive/HiveImport.java index ce34286..838f083 100644 --- a/src/java/org/apache/sqoop/hive/HiveImport.java +++ b/src/java/org/apache/sqoop/hive/HiveImport.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.Tool; import org.apache.sqoop.io.CodecMap; @@ -82,18 +83,19 @@ public class HiveImport { // Fall back to just plain 'hive' and hope it's in the path. String hiveHome = options.getHiveHome(); + String hiveCommand = Shell.WINDOWS ? "hive.cmd" : "hive"; if (null == hiveHome) { - return "hive"; + return hiveCommand; } Path p = new Path(hiveHome); p = new Path(p, "bin"); - p = new Path(p, "hive"); + p = new Path(p, hiveCommand); String hiveBinStr = p.toString(); if (new File(hiveBinStr).exists()) { return hiveBinStr; } else { - return "hive"; + return hiveCommand; } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/java/org/apache/sqoop/orm/CompilationManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java index 92effb5..371ecab 100644 --- a/src/java/org/apache/sqoop/orm/CompilationManager.java +++ b/src/java/org/apache/sqoop/orm/CompilationManager.java @@ -37,6 +37,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.util.Shell; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.util.FileListing; @@ -289,6 +290,11 @@ public class CompilationManager { if (include) { // include this file. + if (Shell.WINDOWS) { + // In Windows OS, elements in jar files still need to have a path + // separator of '/' rather than the default File.separator which is '\' + chompedPath = chompedPath.replace(File.separator, "/"); + } LOG.debug("Got classfile: " + entry.getPath() + " -> " + chompedPath); ZipEntry ze = new ZipEntry(chompedPath); jstream.putNextEntry(ze); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/java/org/apache/sqoop/util/ClassLoaderStack.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/ClassLoaderStack.java b/src/java/org/apache/sqoop/util/ClassLoaderStack.java index 8e41cb3..a90d4a6 100644 --- a/src/java/org/apache/sqoop/util/ClassLoaderStack.java +++ b/src/java/org/apache/sqoop/util/ClassLoaderStack.java @@ -75,7 +75,7 @@ public final class ClassLoaderStack { } } - String urlPath = "jar:file://" + new File(jarFile).getAbsolutePath() + "!/"; + String urlPath = "jar:" + new File(jarFile).toURI().toURL() + "!/"; LOG.debug("Attempting to load jar through URL: " + urlPath); LOG.debug("Previous classloader is " + prevClassLoader); URL [] jarUrlArray = {new URL(urlPath)}; http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/test/com/cloudera/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index 170bc66..462ccf1 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -188,9 +188,10 @@ public class TestHiveImport extends ImportJobTestCase { SqoopOptions options = getSqoopOptions(args, tool); String hiveHome = options.getHiveHome(); assertNotNull("hive.home was not set", hiveHome); - Path testDataPath = new Path(new Path(hiveHome), - "scripts/" + verificationScript); - System.setProperty("expected.script", testDataPath.toString()); + String testDataPath = new Path(new Path(hiveHome), + "scripts/" + verificationScript).toString(); + System.setProperty("expected.script", + new File(testDataPath).getAbsolutePath()); // verify that we can import it correctly into hive. runImport(tool, args); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/test/com/cloudera/sqoop/io/TestNamedFifo.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/io/TestNamedFifo.java b/src/test/com/cloudera/sqoop/io/TestNamedFifo.java index 9f0a585..40f9b3b 100644 --- a/src/test/com/cloudera/sqoop/io/TestNamedFifo.java +++ b/src/test/com/cloudera/sqoop/io/TestNamedFifo.java @@ -36,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; /** * Test the named fifo utility. @@ -158,6 +159,13 @@ public class TestNamedFifo extends TestCase { public void testNamedFifo() throws Exception { + if (Shell.WINDOWS) { + // NamedFifo uses Linux specific commands like mknod + // and mkfifo, so skip the test on Windows OS + LOG.warn("Named FIFO is not supported on Windows. Skipping test"); + return; + } + File root = new File(TEMP_BASE_DIR.toString()); File fifo = new File(root, "foo-fifo"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/test/com/cloudera/sqoop/lib/TestBlobRef.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestBlobRef.java b/src/test/com/cloudera/sqoop/lib/TestBlobRef.java index 2054a9b..d19b769 100644 --- a/src/test/com/cloudera/sqoop/lib/TestBlobRef.java +++ b/src/test/com/cloudera/sqoop/lib/TestBlobRef.java @@ -88,7 +88,7 @@ public class TestBlobRef extends TestCase { FileSystem fs = FileSystem.getLocal(conf); String tmpDir = System.getProperty("test.build.data", "/tmp/"); Path lobDir = new Path(new Path(tmpDir), "_lob"); - fs.delete(lobDir, false); + fs.delete(lobDir, true); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/test/com/cloudera/sqoop/lib/TestClobRef.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestClobRef.java b/src/test/com/cloudera/sqoop/lib/TestClobRef.java index 8cb05e9..7e961c0 100644 --- a/src/test/com/cloudera/sqoop/lib/TestClobRef.java +++ b/src/test/com/cloudera/sqoop/lib/TestClobRef.java @@ -102,7 +102,7 @@ public class TestClobRef extends TestCase { FileSystem fs = FileSystem.getLocal(conf); String tmpDir = System.getProperty("test.build.data", "/tmp/"); Path lobDir = new Path(new Path(tmpDir), "_lob"); - fs.delete(lobDir, false); + fs.delete(lobDir, true); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/src/test/com/cloudera/sqoop/orm/TestClassWriter.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java index bfb25b0..6158fb4 100644 --- a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java +++ b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java @@ -33,6 +33,7 @@ import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -173,6 +174,11 @@ public class TestClassWriter extends TestCase { // check that the .class file made it into the .jar by enumerating // available entries in the jar file. boolean foundCompiledClass = false; + if (Shell.WINDOWS) { + // In Windows OS, elements in jar files still need to have a path + // separator of '/' rather than the default File.separator which is '\' + classFileNameToCheck = classFileNameToCheck.replace(File.separator, "/"); + } try { JarInputStream jis = new JarInputStream(new FileInputStream(jarFile)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f5006b60/testdata/hive/bin/hive.cmd ---------------------------------------------------------------------- diff --git a/testdata/hive/bin/hive.cmd b/testdata/hive/bin/hive.cmd new file mode 100644 index 0000000..5af6330 --- /dev/null +++ b/testdata/hive/bin/hive.cmd @@ -0,0 +1,112 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@rem This is a mock "Hive" shell that validates whether various test imports +@rem succeeded. It accepts commands of the form 'hive -f scriptname' +@rem and validates that the script contents match those of an expected script. +@rem The filename to that expected script is set via the environment variable +@rem EXPECTED_SCRIPT. + +@rem The script will contain a pathname as part of the LOAD DATA INPATH statement; +@rem depending on where you run the tests from, this can change. So the expected +@rem script file actually contains the marker string "BASEPATH" which is replaced +@rem by this script with the contents of $TMPDIR, which is set to 'test.build.data'. + +setlocal enabledelayedexpansion + +@rem Reset ERRORLEVEL if previously set +set ERRORLEVEL= + +if "%EXPECTED_SCRIPT%" =="" ( + echo No expected script set + set ERRORLEVEL=1 + goto :end +) + +if "%TMPDIR%" == "" ( + echo setting TMPDIR to %TEMP% + set TMPDIR=%TEMP% +) + +if not exist %TMPDIR% ( + md %TMPDIR% +) + +if "%1" NEQ "-f" ( + echo "Misunderstood argument: %1" + echo "Expected '-f'." + set ERRORLEVEL=2 + goto :end +) + +if "%2" =="" ( + echo "Expected: hive -f filename" + set ERRORLEVEL=3 + goto :end +) + +set GENERATED_SCRIPT=%2 + +rem Generate the directory temporarily hosting copied base scripts before processing +set TMP_COPY_DIR=%TMPDIR%\tmp +if not exist %TMP_COPY_DIR% ( + md "%TMP_COPY_DIR%" +) + +pushd +rem Adjust path format for TMP_COPY_DIR and TMPDIR +cd /d %TMP_COPY_DIR% +set TMP_COPY_DIR=%cd% + +cd /d %TMPDIR% +set TMPDIR=%cd% +popd + +@rem Copy the expected script into the tmpdir and replace the marker. +copy "%EXPECTED_SCRIPT%" "%TMP_COPY_DIR%" +for %%i in ("%EXPECTED_SCRIPT%") do set SCRIPT_BASE=%%~nxi +set COPIED_SCRIPT=%TMP_COPY_DIR%\%SCRIPT_BASE% + +@rem Replace the BASEPATH marker with actual base-path +set RESOLVED_BASE=/%TMPDIR:\=/% + +@rem Trim all comments from copied base scripts +set BASE_SCRIPT=%TMPDIR%\%SCRIPT_BASE% +if exist %BASE_SCRIPT% ( + del %BASE_SCRIPT% +) + +@rem Filter all comments from the copied script +FOR /F "usebackq delims=" %%i in (%COPIED_SCRIPT%) do ( + set "line=%%i" + set "start=!line:~0,2!" + if not "!start!" == "--" ( + set "resolvedLine=!line:BASEPATH=%RESOLVED_BASE%!" + echo(!resolvedLine!>>!BASE_SCRIPT! + ) +) + +@rem Delete the temporary folder for copied scripts +rd /S /Q %TMP_COPY_DIR% + +rem Actually check to see that the input we got matches up. +fc %BASE_SCRIPT% %GENERATED_SCRIPT% /W + +:end +echo Exiting with return code %ERRORLEVEL% +exit %ERRORLEVEL% +endlocal
