Author: jdere
Date: Fri Apr 4 23:52:40 2014
New Revision: 1584943
URL: http://svn.apache.org/r1584943
Log:
HIVE-5176: Wincompat : Changes for allowing various path compatibilities with
Windows (Jason Dere, reviewed by Thejas Nair)
Added:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
Modified:
hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
Modified:
hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java?rev=1584943&r1=1584942&r2=1584943&view=diff
==============================================================================
--- hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
(original)
+++ hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
Fri Apr 4 23:52:40 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.HiveTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -35,7 +36,13 @@ public class TestHiveConf {
@Test
public void testHiveSitePath() throws Exception {
String expectedPath = HiveTestUtils.getFileFromClasspath("hive-site.xml");
- Assert.assertEquals(expectedPath, new
HiveConf().getHiveSiteLocation().getPath());
+ String hiveSiteLocation = new HiveConf().getHiveSiteLocation().getPath();
+ if (Shell.WINDOWS) {
+ // Do case-insensitive comparison on Windows, as drive letter can have
different case.
+ expectedPath = expectedPath.toLowerCase();
+ hiveSiteLocation = hiveSiteLocation.toLowerCase();
+ }
+ Assert.assertEquals(expectedPath, hiveSiteLocation);
}
private void checkHadoopConf(String name, String expectedHadoopVal) throws
Exception {
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java?rev=1584943&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
(added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java Fri
Apr 4 23:52:40 2014
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.util.Shell;
+
+public class WindowsPathUtil {
+
+ public static void convertPathsFromWindowsToHdfs(HiveConf conf){
+ String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
getHdfsUriString(orgWarehouseDir));
+
+ String orgTestTempDir = System.getProperty("test.tmp.dir");
+ System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir));
+
+ String orgTestDataDir = System.getProperty("test.src.data.dir");
+ System.setProperty("test.src.data.dir", getHdfsUriString(orgTestDataDir));
+
+ String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);
+ conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));
+ }
+
+ private static String getHdfsUriString(String uriStr) {
+ assert uriStr != null;
+ if(Shell.WINDOWS) {
+ // If the URI conversion is from Windows to HDFS then replace the '\'
with '/'
+ // and remove the windows single drive letter & colon from absolute path.
+ return uriStr.replace('\\', '/')
+ .replaceFirst("/[c-zC-Z]:", "/")
+ .replaceFirst("^[c-zC-Z]:", "");
+ }
+ return uriStr;
+ }
+}
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1584943&r1=1584942&r2=1584943&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
Fri Apr 4 23:52:40 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.WindowsPathUtil;
import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
@@ -59,6 +60,7 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.util.Shell;
/**
* Mimics the actual query compiler in generating end to end plans and testing
@@ -80,6 +82,11 @@ public class TestExecDriver extends Test
conf = new HiveConf(ExecDriver.class);
SessionState.start(conf);
+ //convert possible incompatible Windows path in config
+ if (Shell.WINDOWS) {
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
+ }
+
fs = FileSystem.get(conf);
if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) {
throw new RuntimeException(tmpdir + " exists but is not a directory");
@@ -161,7 +168,8 @@ public class TestExecDriver extends Test
}
FSDataInputStream fi_test = fs.open((fs.listStatus(di_test))[0].getPath());
- if (!Utilities.contentsEqual(fi_gold, fi_test, false)) {
+ boolean ignoreWhitespace = Shell.WINDOWS;
+ if (!Utilities.contentsEqual(fi_gold, fi_test, ignoreWhitespace)) {
LOG.error(di_test.toString() + " does not match " + datafile);
assertEquals(false, true);
}
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java?rev=1584943&r1=1584942&r2=1584943&view=diff
==============================================================================
---
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
(original)
+++
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
Fri Apr 4 23:52:40 2014
@@ -34,8 +34,10 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.WindowsPathUtil;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.util.Shell;
import org.apache.thrift.TException;
/**
@@ -61,6 +63,9 @@ public class TestHiveMetaStoreChecker ex
protected void setUp() throws Exception {
super.setUp();
hive = Hive.get();
+ if (Shell.WINDOWS) {
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(hive.getConf());
+ }
checker = new HiveMetaStoreChecker(hive);
partCols = new ArrayList<FieldSchema>();
