Author: daijy
Date: Fri Dec 20 21:56:10 2013
New Revision: 1552815
URL: http://svn.apache.org/r1552815
Log:
HIVE-5957: Fix HCatalog Unit tests on Windows
Modified:
hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java
Modified:
hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
(original)
+++
hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
Fri Dec 20 21:56:10 2013
@@ -21,6 +21,7 @@ package org.apache.hive.hcatalog.common;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
@@ -645,4 +646,10 @@ public class HCatUtil {
return true;
return false;
}
+ /**
+ * Used by various tests to make sure the path is safe for Windows
+ */
+ public static String makePathASafeFileName(String filePath) {
+ return new File(filePath).getPath().replaceAll("\\\\", "/");
+ }
}
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java
Fri Dec 20 21:56:10 2013
@@ -41,8 +41,7 @@ import java.io.IOException;
*/
public class HCatBaseTest {
protected static final Logger LOG =
LoggerFactory.getLogger(HCatBaseTest.class);
- protected static final String TEST_DATA_DIR = System.getProperty("user.dir")
+
- "/build/test/data/" + HCatBaseTest.class.getCanonicalName();
+ protected static final String TEST_DATA_DIR =
org.apache.hive.hcatalog.mapreduce.HCatBaseTest.TEST_DATA_DIR;
protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR +
"/warehouse";
protected HiveConf hiveConf = null;
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
Fri Dec 20 21:56:10 2013
@@ -79,12 +79,12 @@ public class TestHCatPartitionPublish {
@BeforeClass
public static void setup() throws Exception {
+ File workDir =
org.apache.hive.hcatalog.mapreduce.TestHCatPartitionPublish.handleWorkDir();
conf.set("yarn.scheduler.capacity.root.queues", "default");
conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
fs = FileSystem.get(conf);
- System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory()
- .toString(), "/logs").getAbsolutePath());
+ System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
// to use MiniMRCluster. MAPREDUCE-2350
mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
@@ -99,6 +99,7 @@ public class TestHCatPartitionPublish {
MetaStoreUtils.startMetaStore(msPort, ShimLoader
.getHadoopThriftAuthBridge());
+ Thread.sleep(10000);
isServerRunning = true;
securityManager = System.getSecurityManager();
System.setSecurityManager(new NoExitSecurityManager());
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
Fri Dec 20 21:56:10 2013
@@ -67,7 +67,7 @@ public class TestMultiOutputFormat {
private static final Logger LOG =
LoggerFactory.getLogger(TestMultiOutputFormat.class);
private static File workDir;
- private static Configuration mrConf = null;
+ private static JobConf mrConf = null;
private static FileSystem fs = null;
private static MiniMRCluster mrCluster = null;
@@ -82,9 +82,8 @@ public class TestMultiOutputFormat {
System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
// to use MiniMRCluster. MAPREDUCE-2350
- mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
- new JobConf(conf));
- mrConf = mrCluster.createJobConf();
+ mrConf = new JobConf(conf);
+ mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
mrConf);
}
private static void createWorkDir() throws IOException {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
Fri Dec 20 21:56:10 2013
@@ -97,4 +97,10 @@ public class HcatTestUtils {
}
}
+ /**
+ * Used by various tests to make sure the path is safe for Windows
+ */
+ public static String makePathASafeFileName(String filePath) {
+ return new File(filePath).getPath().replaceAll("\\\\", "/");
+ }
}
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
Fri Dec 20 21:56:10 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.pig.PigServer;
import org.junit.Assert;
import org.junit.Before;
@@ -40,8 +41,8 @@ import java.io.IOException;
*/
public class HCatBaseTest {
protected static final Logger LOG =
LoggerFactory.getLogger(HCatBaseTest.class);
- protected static final String TEST_DATA_DIR =
- "/tmp/build/test/data/" + HCatBaseTest.class.getCanonicalName();
+ public static final String TEST_DATA_DIR =
HCatUtil.makePathASafeFileName(System.getProperty("user.dir") +
+ "/build/test/data/" + HCatBaseTest.class.getCanonicalName() + "-" +
System.currentTimeMillis());
protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR +
"/warehouse";
protected HiveConf hiveConf = null;
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
Fri Dec 20 21:56:10 2013
@@ -76,13 +76,17 @@ public class TestHCatPartitionPublish {
private static SecurityManager securityManager;
private static Configuration conf = new Configuration(true);
- @BeforeClass
- public static void setup() throws Exception {
- String testDir = System.getProperty("test.tmp.dir", "./");
+ public static File handleWorkDir() throws IOException {
+ String testDir = System.getProperty("test.data.dir", "./");
testDir = testDir + "/test_hcat_partitionpublish_" + Math.abs(new
Random().nextLong()) + "/";
File workDir = new File(new File(testDir).getCanonicalPath());
FileUtil.fullyDelete(workDir);
workDir.mkdirs();
+ return workDir;
+ }
+ @BeforeClass
+ public static void setup() throws Exception {
+ File workDir = handleWorkDir();
conf.set("yarn.scheduler.capacity.root.queues", "default");
conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java
Fri Dec 20 21:56:10 2013
@@ -64,7 +64,7 @@ public class TestMultiOutputFormat {
private static final Logger LOG =
LoggerFactory.getLogger(TestMultiOutputFormat.class);
private static File workDir;
- private static Configuration mrConf = null;
+ private static JobConf mrConf = null;
private static FileSystem fs = null;
private static MiniMRCluster mrCluster = null;
@@ -79,9 +79,8 @@ public class TestMultiOutputFormat {
System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
// to use MiniMRCluster. MAPREDUCE-2350
- mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
- new JobConf(conf));
- mrConf = mrCluster.createJobConf();
+ mrConf = new JobConf(conf);
+ mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
mrConf);
}
private static void createWorkDir() throws IOException {
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
Fri Dec 20 21:56:10 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hcatalog.HcatTestUtils;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.data.Pair;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.ResourceStatistics;
@@ -60,8 +61,9 @@ import org.junit.Test;
* @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatLoader}
instead
*/
public class TestHCatLoader {
- private static final String TEST_DATA_DIR =
System.getProperty("java.io.tmpdir") + File.separator
- + TestHCatLoader.class.getCanonicalName() + "-" +
System.currentTimeMillis();
+ private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(
+ System.getProperty("java.io.tmpdir") + File.separator +
TestHCatLoader.class.getCanonicalName() + "-" +
+ System.currentTimeMillis());
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR +
"/warehouse";
private static final String BASIC_FILE_NAME = TEST_DATA_DIR +
"/basic.input.data";
private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR +
"/complex.input.data";
@@ -409,8 +411,8 @@ public class TestHCatLoader {
assertEquals(0, driver.run("drop table if exists " +
tbl).getResponseCode());
assertEquals(0, driver.run("create external table " + tbl +
" (a string, b boolean) row format delimited fields terminated by '\t'" +
- " stored as textfile location 'file://" +
- inputDataDir.getAbsolutePath() + "'").getResponseCode());
+ " stored as textfile location 'file:///" +
+ inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode());
Properties properties = new Properties();
properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
"true");
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java
Fri Dec 20 21:56:10 2013
@@ -69,7 +69,7 @@ public class TestHCatLoaderStorer extend
" (my_small_int smallint, my_tiny_int tinyint)" +
" row format delimited fields terminated by '\t' stored as
textfile").getResponseCode());
Assert.assertEquals(0, driver.run("load data local inpath '" +
- dataDir.getAbsolutePath() + "' into table " +
readTblName).getResponseCode());
+ dataDir.getPath().replaceAll("\\\\", "/") + "' into table " +
readTblName).getResponseCode());
PigServer server = new PigServer(ExecType.LOCAL);
server.registerQuery(
@@ -104,7 +104,7 @@ public class TestHCatLoaderStorer extend
String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE),
String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE)
});
- smallTinyIntBoundsCheckHelper(writeDataFile.getAbsolutePath(),
ExecJob.JOB_STATUS.COMPLETED);
+ smallTinyIntBoundsCheckHelper(writeDataFile.getPath().replaceAll("\\\\",
"/"), ExecJob.JOB_STATUS.COMPLETED);
// Values outside the column type bounds will fail at runtime.
HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv", new
String[]{
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java
Fri Dec 20 21:56:10 2013
@@ -41,8 +41,7 @@ import org.apache.pig.PigServer;
* @deprecated Use/modify {@link
org.apache.hive.hcatalog.pig.TestHCatStorerMulti} instead
*/
public class TestHCatStorerMulti extends TestCase {
- private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
- "/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName();
+ private static final String TEST_DATA_DIR =
org.apache.hive.hcatalog.pig.TestHCatStorerMulti.TEST_DATA_DIR;
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR +
"/warehouse";
private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java
Fri Dec 20 21:56:10 2013
@@ -48,7 +48,7 @@ public class TestHCatStorerWrapper exten
@Test
public void testStoreExternalTableWithExternalDir() throws IOException,
CommandNeedRetryException{
- File tmpExternalDir = new File(SystemUtils.getJavaIoTmpDir(),
UUID.randomUUID().toString());
+ File tmpExternalDir = new File(TEST_DATA_DIR,
UUID.randomUUID().toString());
tmpExternalDir.deleteOnExit();
String part_val = "100";
@@ -71,11 +71,11 @@ public class TestHCatStorerWrapper exten
server.setBatchOn();
logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int,
b:chararray);");
logAndRegister(server, "store A into 'default.junit_external' using " +
HCatStorerWrapper.class.getName()
- + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() +
"');");
+ + "('c=" + part_val + "','" +
tmpExternalDir.getPath().replaceAll("\\\\", "/") + "');");
server.executeBatch();
Assert.assertTrue(tmpExternalDir.exists());
- Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" +
"part-m-00000").exists());
+ Assert.assertTrue(new File(tmpExternalDir.getPath().replaceAll("\\\\",
"/") + "/" + "part-m-00000").exists());
driver.run("select * from junit_external");
ArrayList<String> res = new ArrayList<String>();
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
Fri Dec 20 21:56:10 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hive.hcatalog.HcatTestUtils;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.data.Pair;
import org.apache.pig.ExecType;
@@ -57,8 +58,8 @@ import org.junit.Before;
import org.junit.Test;
public class TestHCatLoader {
- private static final String TEST_DATA_DIR =
System.getProperty("java.io.tmpdir") + File.separator
- + TestHCatLoader.class.getCanonicalName() + "-" +
System.currentTimeMillis();
+ private static final String TEST_DATA_DIR =
HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
+ File.separator + TestHCatLoader.class.getCanonicalName() + "-" +
System.currentTimeMillis());
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR +
"/warehouse";
private static final String BASIC_FILE_NAME = TEST_DATA_DIR +
"/basic.input.data";
private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR +
"/complex.input.data";
@@ -420,7 +421,7 @@ public class TestHCatLoader {
assertEquals(0, driver.run("drop table if exists " +
tbl).getResponseCode());
assertEquals(0, driver.run("create external table " + tbl +
" (a string, b boolean) row format delimited fields terminated by '\t'" +
- " stored as textfile location 'file://" +
+ " stored as textfile location 'file:///" +
inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode());
Properties properties = new Properties();
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
Fri Dec 20 21:56:10 2013
@@ -33,13 +33,15 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.Pair;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
public class TestHCatStorerMulti extends TestCase {
- private static final String TEST_DATA_DIR =
- "/tmp/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName();
+ public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(
+ System.getProperty("user.dir") + "/build/test/data/" +
+ TestHCatStorerMulti.class.getCanonicalName() + "-" +
System.currentTimeMillis());
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR +
"/warehouse";
private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
Modified:
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
(original)
+++
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
Fri Dec 20 21:56:10 2013
@@ -165,12 +165,8 @@ public abstract class SkeletonHBaseTest
protected int usageCount = 0;
public Context(String handle) {
- try {
- testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new
Random().nextLong()) + "/").getCanonicalPath();
- System.out.println("Cluster work directory: " + testDir);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to generate testDir", e);
- }
+ testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new
Random().nextLong()) + "/").getPath();
+ System.out.println("Cluster work directory: " + testDir);
}
public void start() {
Modified:
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
(original)
+++
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
Fri Dec 20 21:56:10 2013
@@ -64,7 +64,7 @@ import org.apache.hcatalog.cli.HCatDrive
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatException;
-import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.HCatRecord;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
@@ -188,7 +188,7 @@ public class TestHCatHBaseInputFormat ex
String databaseName = newTableName("MyDatabase");
//Table name will be lower case unless specified by hbase.table.name
property
String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
- String db_dir = getTestDir() + "/hbasedb";
+ String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + "
LOCATION '"
+ db_dir + "'";
Modified:
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java
(original)
+++
hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java
Fri Dec 20 21:56:10 2013
@@ -172,12 +172,8 @@ public abstract class SkeletonHBaseTest
protected int usageCount = 0;
public Context(String handle) {
- try {
- testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new
Random().nextLong()) + "/").getCanonicalPath();
- System.out.println("Cluster work directory: " + testDir);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to generate testDir", e);
- }
+ testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new
Random().nextLong()) + "/").getPath();
+ System.out.println("Cluster work directory: " + testDir);
}
public void start() {
Modified:
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
(original)
+++
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
Fri Dec 20 21:56:10 2013
@@ -128,7 +128,8 @@ public class TestHCatClient {
assertTrue(testDb.getProperties().size() == 0);
String warehouseDir = System
.getProperty("test.warehouse.dir", "/user/hive/warehouse");
- String expectedDir = warehouseDir.replaceAll("\\\\",
"/").replaceFirst("pfile:///", "pfile:/");
+ String expectedDir =
org.apache.hive.hcatalog.api.TestHCatClient.fixPath(warehouseDir).
+ replaceFirst("pfile:///", "pfile:/");
assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation());
ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
cols.add(new HCatFieldSchema("id", Type.INT, "id comment"));
Modified:
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
(original)
+++
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
Fri Dec 20 21:56:10 2013
@@ -53,6 +53,8 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertArrayEquals;
+import org.apache.hadoop.util.Shell;
+
public class TestHCatClient {
private static final Logger LOG =
LoggerFactory.getLogger(TestHCatClient.class);
private static final String msPort = "20101";
@@ -100,7 +102,16 @@ public class TestHCatClient {
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
}
-
+ public static String fixPath(String path) {
+ if(!Shell.WINDOWS) {
+ return path;
+ }
+ String expectedDir = path.replaceAll("\\\\", "/");
+ if (!expectedDir.startsWith("/")) {
+ expectedDir = "/" + expectedDir;
+ }
+ return expectedDir;
+ }
@Test
public void testBasicDDLCommands() throws Exception {
String db = "testdb";
@@ -121,7 +132,7 @@ public class TestHCatClient {
assertTrue(testDb.getProperties().size() == 0);
String warehouseDir = System
.getProperty("test.warehouse.dir", "/user/hive/warehouse");
- String expectedDir = warehouseDir.replaceAll("\\\\",
"/").replaceFirst("pfile:///", "pfile:/");
+ String expectedDir = fixPath(warehouseDir).replaceFirst("pfile:///",
"pfile:/");
assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation());
ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
cols.add(new HCatFieldSchema("id", Type.INT, "id comment"));
Modified:
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
(original)
+++
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
Fri Dec 20 21:56:10 2013
@@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hcatalog.HcatTestUtils;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatException;
-import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.DefaultHCatRecord;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
@@ -72,8 +72,8 @@ public class TestSequenceFileReadWrite {
dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator +
TestSequenceFileReadWrite.class.getCanonicalName() + "-" +
System.currentTimeMillis());
hiveConf = new HiveConf(this.getClass());
- warehouseDir = new File(dataDir, "warehouse").getAbsolutePath();
- inputFileName = new File(dataDir, "input.data").getAbsolutePath();
+ warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator +
"warehouse");
+ inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator +
"input.data");
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
Modified:
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java
(original)
+++
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java
Fri Dec 20 21:56:10 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.DataType;
@@ -138,7 +139,7 @@ public class TestPigHBaseStorageHandler
String databaseName = newTableName("MyDatabase");
//Table name will be lower case unless specified by hbase.table.name
property
String hbaseTableName = "testTable";
- String db_dir = getTestDir() + "/hbasedb";
+ String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + "
LOCATION '"
+ db_dir + "'";
@@ -193,7 +194,7 @@ public class TestPigHBaseStorageHandler
String databaseName = newTableName("MyDatabase");
//Table name will be lower case unless specified by hbase.table.name
property
String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
- String db_dir = getTestDir() + "/hbasedb";
+ String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + "
LOCATION '"
+ db_dir + "'";
@@ -255,7 +256,7 @@ public class TestPigHBaseStorageHandler
String databaseName = newTableName("MyDatabase");
//Table name will be lower case unless specified by hbase.table.name
property
String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
- String db_dir = getTestDir() + "/hbasedb";
+ String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
String POPTXT_FILE_NAME = db_dir+"testfile.txt";
float f = -100.1f;
Modified:
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1552815&r1=1552814&r2=1552815&view=diff
==============================================================================
---
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java
(original)
+++
hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java
Fri Dec 20 21:56:10 2013
@@ -69,8 +69,8 @@ public class TestSequenceFileReadWrite {
dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator +
TestSequenceFileReadWrite.class.getCanonicalName() + "-" +
System.currentTimeMillis());
hiveConf = new HiveConf(this.getClass());
- warehouseDir = new File(dataDir, "warehouse").getAbsolutePath();
- inputFileName = new File(dataDir, "input.data").getAbsolutePath();
+ warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator +
"warehouse");
+ inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator +
"input.data");
hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
