Repository: hive Updated Branches: refs/heads/master 99cb7c005 -> 77c145043
HIVE-19788: Flaky test: HCatalog-PigServer tests (Sahil Takiar, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77c14504 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77c14504 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77c14504 Branch: refs/heads/master Commit: 77c1450430d8ed7b14d7c3c36794a0467c9571db Parents: 99cb7c0 Author: Sahil Takiar <takiar.sa...@gmail.com> Authored: Mon Jun 4 17:03:39 2018 -0500 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Wed Jun 6 15:58:49 2018 -0500 ---------------------------------------------------------------------- .../hive/hcatalog/mapreduce/HCatBaseTest.java | 8 +++-- .../hcatalog/mapreduce/TestPassProperties.java | 4 --- .../hcatalog/pig/AbstractHCatLoaderTest.java | 5 ++-- .../hcatalog/pig/AbstractHCatStorerTest.java | 31 ++++++++++---------- .../hive/hcatalog/pig/TestE2EScenarios.java | 4 +-- .../pig/TestHCatLoaderComplexSchema.java | 4 +-- .../hcatalog/pig/TestHCatLoaderEncryption.java | 6 ++-- .../hive/hcatalog/pig/TestHCatLoaderStorer.java | 5 ++-- .../hive/hcatalog/pig/TestHCatStorerMulti.java | 9 +++--- .../hcatalog/pig/TestHCatStorerWrapper.java | 3 +- .../hbase/TestPigHBaseStorageHandler.java | 8 ++--- .../mapreduce/TestHCatHiveCompatibility.java | 5 ++-- .../TestHCatHiveThriftCompatibility.java | 3 +- .../mapreduce/TestSequenceFileReadWrite.java | 2 +- 14 files changed, 45 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java index 8471e5b..a304e49 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java @@ -109,14 +109,18 @@ public abstract class HCatBaseTest { server.registerQuery(query, lineNumber); } + public static PigServer createPigServer(boolean stopOnFailure) throws ExecException { + return createPigServer(stopOnFailure, new Properties()); + } + /** * creates PigServer in LOCAL mode. * http://pig.apache.org/docs/r0.12.0/perf.html#error-handling * @param stopOnFailure equivalent of "-stop_on_failure" command line arg, setting to 'true' makes * debugging easier */ - public static PigServer createPigServer(boolean stopOnFailure) throws ExecException { - Properties p = new Properties(); + public static PigServer createPigServer(boolean stopOnFailure, Properties p) throws + ExecException { Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); String testId = "HCatBaseTest_" + System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java index 1df3c24..332c240 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java @@ -44,8 +44,6 @@ import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.DefaultHCatRecord; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.pig.ExecType; -import org.apache.pig.PigServer; import org.junit.Test; public class TestPassProperties { @@ -55,7 +53,6 @@ public class TestPassProperties { private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; private static IDriver driver; - private static PigServer server; private static String[] input; private static HiveConf hiveConf; @@ -81,7 +78,6 @@ public class TestPassProperties { input[i] = i + "," + col1 + "," + col2; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - server = new PigServer(ExecType.LOCAL); } @Test http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java index 14b22ed..0d72102 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java @@ -52,7 +52,6 @@ import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; -import org.apache.pig.ExecType; import org.apache.pig.PigRunner; import org.apache.pig.PigServer; import org.apache.pig.ResourceStatistics; @@ -584,7 +583,7 @@ public abstract class AbstractHCatLoaderTest extends HCatBaseTest { Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "true"); properties.put("stop.on.failure", Boolean.TRUE.toString()); - PigServer server = new PigServer(ExecType.LOCAL, properties); + PigServer server = createPigServer(true, properties); server.registerQuery( "data = load 'test_convert_boolean_to_int' using org.apache.hive.hcatalog.pig.HCatLoader();"); Schema schema = server.dumpSchema("data"); @@ -689,7 +688,7 @@ public abstract class AbstractHCatLoaderTest extends HCatBaseTest { // might be the last one to call HCatContext.INSTANCE.setConf(). Make sure setting is false. Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "false"); - PigServer server = new PigServer(ExecType.LOCAL, properties); + PigServer server = createPigServer(false, properties); server.registerQuery("X = load '" + ALL_PRIMITIVE_TYPES_TABLE + "' using " + HCatLoader.class.getName() + "();"); Iterator<Tuple> XIter = server.openIterator("X"); int numTuplesRead = 0; http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java index 30b0047..97277b5 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.EvalFunc; -import org.apache.pig.ExecType; import org.apache.pig.PigException; import org.apache.pig.PigServer; import org.apache.pig.data.DataByteArray; @@ -425,7 +424,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { input[i] = i + "\t1"; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('b=1');"); @@ -460,7 +459,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { "111239\tSatya\t01/01/2001\tM\tIN\tKL", "111240\tKavya\t01/01/2002\tF\tIN\tAP" }; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -511,7 +510,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { input[i] = i + ""; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int);"); server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('b=1');"); @@ -541,7 +540,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { public void testNoAlias() throws Exception { AbstractHCatLoaderTest.dropTable("junit_parted", driver); AbstractHCatLoaderTest.createTable("junit_parted","a int, b string", "ds string", driver, storageFormat); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); boolean errCaught = false; try { server.setBatchOn(); @@ -601,7 +600,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("B = filter A by a < 2;"); @@ -648,7 +647,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'default.junit_unparted' using " @@ -684,7 +683,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'junit_unparted' using " + HCatStorer.class.getName() @@ -720,7 +719,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("B = filter A by a > 100;"); @@ -751,7 +750,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server .registerQuery("A = load '" @@ -797,7 +796,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { + "\tbinary-data"; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray);"); @@ -854,7 +853,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'default.junit_unparted' using " @@ -888,7 +887,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { "111239\tSatya\t01/01/2001\tM\tIN\tKL", "111240\tKavya\t01/01/2002\tF\tIN\tAP" }; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -921,7 +920,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { "111239\tSatya\t01/01/2001\tM\tIN\tKL", "111240\tKavya\t01/01/2002\tF\tIN\tAP" }; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -952,7 +951,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { String[] inputData = {}; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -980,7 +979,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { input[i] = i + "\tmath"; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, c:chararray);"); server.registerQuery("B = filter A by " + FailEvalFunc.class.getName() + "($0);"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java index e0f1c89..19246e6 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java @@ -47,12 +47,12 @@ import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatContext; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; @@ -125,7 +125,7 @@ public class TestE2EScenarios { } private void pigDump(String tableName) throws IOException { - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); System.err.println("==="); System.err.println(tableName+":"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index eedd722..8f06d39 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.pig.ExecType; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.executionengine.ExecJob; @@ -224,7 +224,7 @@ public class TestHCatLoaderComplexSchema { MockLoader.setData(tablename + "Input", data); try { createTable(tablename, tableSchema); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + tablename + "Input' using org.apache.hive.hcatalog.pig.MockLoader() AS (" + pigSchema + ");"); Schema dumpedASchema = server.dumpSchema("A"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index 72ed8df..872d7af 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -65,8 +65,8 @@ import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.Pair; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; import org.junit.After; @@ -206,7 +206,7 @@ public class TestHCatLoaderEncryption { } } HcatTestUtils.createTestDataFile(BASIC_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); int i = 0; server.registerQuery("A = load '" + BASIC_FILE_NAME + "' as (a:int, b:chararray);", ++i); @@ -300,7 +300,7 @@ public class TestHCatLoaderEncryption { @Test public void testReadDataFromEncryptedHiveTableByPig() throws IOException { assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.registerQuery("X = load '" + ENCRYPTED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();"); Iterator<Tuple> XIter = server.openIterator("X"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java index ae40f0b..c7f2606 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java @@ -22,7 +22,6 @@ package org.apache.hive.hcatalog.pig; import org.apache.hadoop.fs.FileUtil; import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob; import org.apache.pig.data.DataType; @@ -127,7 +126,7 @@ public class TestHCatLoaderStorer extends HCatBaseTest { Assert.assertEquals(0, driver.run("load data local inpath '" + dataDir.getPath().replaceAll("\\\\", "/") + "' into table " + readTblName).getResponseCode()); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.registerQuery( "data = load '" + readTblName + "' using org.apache.hive.hcatalog.pig.HCatLoader();"); @@ -186,7 +185,7 @@ public class TestHCatLoaderStorer extends HCatBaseTest { Assert.assertEquals(0, driver.run("create table test_tbl" + " (my_small_int smallint, my_tiny_int tinyint) stored as rcfile").getResponseCode()); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("data = load '" + data + "' using PigStorage('\t') as (my_small_int:int, my_tiny_int:int);"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index d6b3ebc..65ad6ec 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -37,8 +37,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; - -import org.apache.pig.ExecType; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.PigServer; import org.junit.After; @@ -123,7 +122,7 @@ public class TestHCatStorerMulti { populateBasicFile(); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();"); @@ -143,7 +142,7 @@ public class TestHCatStorerMulti { populateBasicFile(); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); @@ -168,7 +167,7 @@ public class TestHCatStorerMulti { populateBasicFile(); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java index 0ffab4a..3edd020 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java @@ -27,7 +27,6 @@ import java.util.UUID; import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.junit.Assert; @@ -66,7 +65,7 @@ public class TestHCatStorerWrapper extends HCatBaseTest { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);"); logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName() http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java index 4778e1f..4219757 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.pig.ExecType; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.PigServer; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; @@ -195,7 +195,7 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { assertTrue(doesTableExist); - PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + PigServer server = HCatBaseTest.createPigServer(false, hcatConf.getAllProperties()); server.registerQuery("A = load '"+databaseName+"."+tableName+"' using org.apache.hive.hcatalog.pig.HCatLoader();"); Schema dumpedASchema = server.dumpSchema("A"); @@ -281,7 +281,7 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { int index=1; - PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + PigServer server = HCatBaseTest.createPigServer(false, hcatConf.getAllProperties()); server.registerQuery("A = load '"+databaseName+"."+tableName+"' using org.apache.hive.hcatalog.pig.HCatLoader();"); server.registerQuery("B = filter A by key < 5;"); server.registerQuery("C = foreach B generate key,testqualifier2;"); @@ -351,7 +351,7 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { createTestDataFile(POPTXT_FILE_NAME); - PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + PigServer server = HCatBaseTest.createPigServer(false, hcatConf.getAllProperties()); server.registerQuery("A = load '"+POPTXT_FILE_NAME+"' using PigStorage() as (key:int, testqualifier1:float, testqualifier2:chararray);"); server.registerQuery("B = filter A by (key > 2) AND (key < 8) ;"); server.registerQuery("store B into '"+databaseName.toLowerCase()+"."+tableName.toLowerCase()+"' using org.apache.hive.hcatalog.pig.HCatStorer();"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java index bb77005..dac6218 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java @@ -28,7 +28,6 @@ import junit.framework.Assert; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; import org.junit.BeforeClass; @@ -60,7 +59,7 @@ public class TestHCatHiveCompatibility extends HCatBaseTest { Table table = client.getTable("default", "junit_unparted_noisd"); Assert.assertTrue(table.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS)); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); logAndRegister(server, "A = load '" + INPUT_FILE_NAME + "' as (a:int);"); logAndRegister(server, "store A into 'default.junit_unparted_noisd' using org.apache.hive.hcatalog.pig.HCatStorer();"); logAndRegister(server, "B = load 'default.junit_unparted_noisd' using org.apache.hive.hcatalog.pig.HCatLoader();"); @@ -96,7 +95,7 @@ public class TestHCatHiveCompatibility extends HCatBaseTest { Table table = client.getTable("default", "junit_parted_noisd"); Assert.assertTrue(table.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS)); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); logAndRegister(server, "A = load '" + INPUT_FILE_NAME + "' as (a:int);"); logAndRegister(server, "store A into 'default.junit_parted_noisd' using org.apache.hive.hcatalog.pig.HCatStorer('b=42');"); logAndRegister(server, "B = load 'default.junit_parted_noisd' using org.apache.hive.hcatalog.pig.HCatLoader();"); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java index a024c83..1c14865 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.serde2.thrift.test.IntString; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; @@ -93,7 +92,7 @@ public class TestHCatHiveThriftCompatibility extends HCatBaseTest { driver.run("alter table test_thrift add partition (year = '2012') location '" + intStringSeq.getParent() + "'").getResponseCode()); - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = createPigServer(false); pigServer.registerQuery("A = load 'test_thrift' using org.apache.hive.hcatalog.pig.HCatLoader();"); Schema expectedSchema = new Schema(); http://git-wip-us.apache.org/repos/asf/hive/blob/77c14504/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java index 4ebec1b..ab25ffe 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java @@ -96,7 +96,7 @@ public class TestSequenceFileReadWrite { input[i] = i + "," + col1 + "," + col2; } HcatTestUtils.createTestDataFile(inputFileName, input); - server = new PigServer(ExecType.LOCAL); + server = HCatBaseTest.createPigServer(false); } @After public void teardown() throws IOException {
