Author: khorgath
Date: Thu May 1 01:08:11 2014
New Revision: 1591527
URL: http://svn.apache.org/r1591527
Log:
HIVE-6931 : Windows unit test fixes (Jason Dere via Sushanth Sowmyan)
Modified:
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/trunk/pom.xml
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q
hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out
Modified:
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
---
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
(original)
+++
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
Thu May 1 01:08:11 2014
@@ -258,7 +258,7 @@ public class QTestUtil {
mr.setupConfiguration(conf);
// set fs.default.name to the uri of mini-dfs
- String dfsUriString =
getHdfsUriString(dfs.getFileSystem().getUri().toString());
+ String dfsUriString =
WindowsPathUtil.getHdfsUriString(dfs.getFileSystem().getUri().toString());
conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString);
// hive.metastore.warehouse.dir needs to be set relative to the mini-dfs
conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
@@ -269,46 +269,10 @@ public class QTestUtil {
// Windows paths should be converted after MiniMrShim.setupConfiguration()
// since setupConfiguration may overwrite configuration values.
if (Shell.WINDOWS) {
- convertPathsFromWindowsToHdfs();
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
}
}
- private void convertPathsFromWindowsToHdfs() {
- // Following local paths are used as HDFS paths in unit tests.
- // It works well in Unix as the path notation in Unix and HDFS is more or
less same.
- // But when it comes to Windows, drive letter separator ':' & backslash
'\" are invalid
- // characters in HDFS so we need to converts these local paths to HDFS
paths before using them
- // in unit tests.
-
- // hive.exec.scratchdir needs to be set relative to the mini-dfs
- String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
- conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
getHdfsUriString(orgWarehouseDir));
-
- String orgTestTempDir = System.getProperty("test.tmp.dir");
- System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir));
-
- String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);
- conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));
-
- if (miniMr) {
- String orgAuxJarFolder = conf.getAuxJars();
- conf.setAuxJars(getHdfsUriString("file://" + orgAuxJarFolder));
- }
- }
-
- private String getHdfsUriString(String uriStr) {
- assert uriStr != null;
- if(Shell.WINDOWS) {
- // If the URI conversion is from Windows to HDFS then replace the '\'
with '/'
- // and remove the windows single drive letter & colon from absolute path.
- return uriStr.replace('\\', '/')
- .replaceFirst("/[c-zC-Z]:", "/")
- .replaceFirst("^[c-zC-Z]:", "");
- }
-
- return uriStr;
- }
-
public enum MiniClusterType {
mr,
tez,
@@ -361,7 +325,7 @@ public class QTestUtil {
if (clusterType != MiniClusterType.none) {
dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
FileSystem fs = dfs.getFileSystem();
- String uriString = getHdfsUriString(fs.getUri().toString());
+ String uriString =
WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
mr = shims.getMiniTezCluster(conf, 4, uriString, 1);
} else {
Modified: hive/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Thu May 1 01:08:11 2014
@@ -64,6 +64,7 @@
<test.hive.hadoop.classpath>${maven.test.classpath}</test.hive.hadoop.classpath>
<test.log4j.scheme>file://</test.log4j.scheme>
<test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir>
+ <test.tmp.dir.uri>file://${test.tmp.dir}</test.tmp.dir.uri>
<test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir>
<test.warehouse.scheme>pfile://</test.warehouse.scheme>
@@ -757,6 +758,7 @@
<test.data.files>${basedir}/${hive.path.to.root}/data/files</test.data.files>
<test.data.dir>${basedir}/${hive.path.to.root}/data/files</test.data.dir>
<test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
+ <test.tmp.dir.uri>${test.tmp.dir.uri}</test.tmp.dir.uri>
<test.dfs.mkdir>${test.dfs.mkdir}</test.dfs.mkdir>
<test.output.overwrite>${test.output.overwrite}</test.output.overwrite>
<test.warehouse.dir>${test.warehouse.scheme}${test.warehouse.dir}</test.warehouse.dir>
@@ -1047,6 +1049,7 @@
<hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop.cmd</hadoop.bin.path>
<test.extra.path>;${env.HADOOP_HOME}/bin</test.extra.path>
<test.hive.hadoop.classpath>${project.build.directory}/deplibs/*</test.hive.hadoop.classpath>
+ <test.tmp.dir.uri>file:///${test.tmp.dir}</test.tmp.dir.uri>
<test.log4j.scheme>file:/</test.log4j.scheme>
</properties>
</profile>
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java Thu
May 1 01:08:11 2014
@@ -24,17 +24,26 @@ import org.apache.hadoop.util.Shell;
public class WindowsPathUtil {
public static void convertPathsFromWindowsToHdfs(HiveConf conf){
+ // Following local paths are used as HDFS paths in unit tests.
+ // It works well in Unix as the path notation in Unix and HDFS is more or
less same.
+ // But when it comes to Windows, drive letter separator ':' & backslash
'\" are invalid
+ // characters in HDFS so we need to converts these local paths to HDFS
paths before using them
+ // in unit tests.
+
String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
getHdfsUriString(orgWarehouseDir));
String orgTestTempDir = System.getProperty("test.tmp.dir");
System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir));
+ String orgTestWarehouseDir = System.getProperty("test.warehouse.dir");
+ System.setProperty("test.warehouse.dir",
getHdfsUriString(orgTestWarehouseDir));
+
String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);
conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));
}
- private static String getHdfsUriString(String uriStr) {
+ public static String getHdfsUriString(String uriStr) {
assert uriStr != null;
if(Shell.WINDOWS) {
// If the URI conversion is from Windows to HDFS then replace the '\'
with '/'
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
Thu May 1 01:08:11 2014
@@ -71,9 +71,9 @@ public class TestExecDriver extends Test
static HiveConf conf;
- private static final String tmpdir = System.getProperty("test.tmp.dir");
+ private static final String tmpdir;
private static final Log LOG = LogFactory.getLog(TestExecDriver.class);
- private static final Path tmppath = new Path(tmpdir);
+ private static final Path tmppath;
private static Hive db;
private static FileSystem fs;
@@ -86,6 +86,8 @@ public class TestExecDriver extends Test
if (Shell.WINDOWS) {
WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
}
+ tmpdir = System.getProperty("test.tmp.dir");
+ tmppath = new Path(tmpdir);
fs = FileSystem.get(conf);
if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) {
Modified: hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q Thu May 1
01:08:11 2014
@@ -3,7 +3,7 @@ set hive.input.format=org.apache.hadoop.
CREATE TABLE dest1(key INT, value STRING);
-ADD FILE src/test/scripts/testgrep_win.bat;
+ADD FILE ../../ql/src/test/scripts/testgrep_win.bat;
FROM (
FROM src
Modified:
hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
---
hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
(original)
+++
hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
Thu May 1 01:08:11 2014
@@ -1,5 +1,5 @@
insert overwrite local directory '${system:test.tmp.dir}/tez_local_src_table_1'
select * from src order by key limit 10 ;
-dfs -cat file:${system:test.tmp.dir}/tez_local_src_table_1/000000_0 ;
+dfs -cat ${system:test.tmp.dir.uri}/tez_local_src_table_1/* ;
-dfs -rmr file:${system:test.tmp.dir}/tez_local_src_table_1/ ;
+dfs -rmr ${system:test.tmp.dir.uri}/tez_local_src_table_1/ ;
Modified: hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out
(original)
+++ hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out Thu May
1 01:08:11 2014
@@ -2,10 +2,12 @@ PREHOOK: query: -- INCLUDE_OS_WINDOWS
CREATE TABLE dest1(key INT, value STRING)
PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
POSTHOOK: query: -- INCLUDE_OS_WINDOWS
CREATE TABLE dest1(key INT, value STRING)
POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
POSTHOOK: Output: default@dest1
PREHOOK: query: FROM (
FROM src
