Modified: pig/branches/spark/test/org/apache/pig/test/TestPigRunner.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPigRunner.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPigRunner.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPigRunner.java Fri Feb 24 03:34:37 2017 @@ -19,7 +19,6 @@ package org.apache.pig.test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -36,8 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.Counters; @@ -65,10 +62,7 @@ import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; public class TestPigRunner { @@ -79,9 +73,6 @@ public class TestPigRunner { private static final String OUTPUT_FILE = "output"; private static final String PIG_FILE = "test.pig"; - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass public static void setUpBeforeClass() throws Exception { cluster = MiniGenericCluster.buildCluster(); @@ -812,9 +803,10 @@ public class TestPigRunner { } @Test - @Ignore - // Skip in hadoop 23 test, see PIG-2449 public void classLoaderTest() throws Exception { + // Skip in hadoop 23 test, see PIG-2449 + if (org.apache.pig.impl.util.Utils.isHadoop23() || org.apache.pig.impl.util.Utils.isHadoop2()) + return; PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE)); w.println("register test/org/apache/pig/test/data/pigtestloader.jar"); w.println("A = load '" + INPUT_FILE + "' using org.apache.pig.test.PigTestLoader();"); @@ -1155,13 +1147,8 @@ public class TestPigRunner { List<OutputStats> outputs = stats.getOutputStats(); assertEquals(2, outputs.size()); if (execType.equals("tez")) { - if( outputs.get(0).getLocation().endsWith("tmp/output") ) { - assertEquals(2, outputs.get(0).getNumberRecords()); - assertEquals(5, outputs.get(1).getNumberRecords()); - } else { - assertEquals(5, outputs.get(0).getNumberRecords()); - assertEquals(2, outputs.get(1).getNumberRecords()); - } + assertEquals(outputs.get(0).getNumberRecords(), 5); + assertEquals(outputs.get(1).getNumberRecords(), 2); } else { for (OutputStats outstats : outputs) { // the multi-output counters are disabled @@ -1227,77 +1214,6 @@ public class TestPigRunner { Util.deleteFile(cluster, "tmp/output"); } } - - @Test - public void testStoredScriptContents() throws Exception { - String scriptContents = "sh echo success;\n"; - FileUtils.writeStringToFile(new File(PIG_FILE), scriptContents); - Util.copyFromLocalToCluster(cluster, PIG_FILE, PIG_FILE); - - Path inputInDfs = new Path(cluster.getFileSystem().getHomeDirectory(), PIG_FILE); - try { - runAndValidateStoredScriptContents(PIG_FILE, scriptContents); - runAndValidateStoredScriptContents(inputInDfs.toString(), scriptContents); - } finally { - FileUtils.deleteQuietly(new File(PIG_FILE)); - Util.deleteQuietly(cluster, PIG_FILE); - } - } - - @Test - public void testErrorLogUnderCustomDir() throws Exception { - try (PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE))) { - w.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);"); - w.println("B = foreach A generate StringSize(a0);"); - w.println("store B into '" + OUTPUT_FILE + "';"); - } - Util.copyFromLocalToCluster(cluster, PIG_FILE, PIG_FILE); - - Path inputInDfs = new Path(cluster.getFileSystem().getHomeDirectory(), PIG_FILE); - try { - runAndValidateCustomErrorLogDir(PIG_FILE); - runAndValidateCustomErrorLogDir(inputInDfs.toString()); - } finally { - FileUtils.deleteQuietly(new File(PIG_FILE)); - Util.deleteQuietly(cluster, PIG_FILE); - } - } - - private void runAndValidateStoredScriptContents(String scriptPath, String expectedContents) { - PigStats stats = runPigLocally(scriptPath); - assertTrue(stats.isSuccessful()); - assertEquals(expectedContents, stats.getScript()); - - stats = runPigLocally("-f", scriptPath); - assertTrue(stats.isSuccessful()); - assertEquals(expectedContents, stats.getScript()); - } - - private void runAndValidateCustomErrorLogDir(String scriptPath) throws IOException { - File logsFolder = temporaryFolder.newFolder(); - String logsPath = logsFolder.getAbsolutePath(); - assertFileCountUnderDir(logsFolder, 0); - - PigStats stats = runPigLocally("-l", logsPath, scriptPath); - assertFalse(stats.isSuccessful()); - assertFileCountUnderDir(logsFolder, 1); - - stats = runPigLocally("-l", logsPath, "-f", scriptPath); - assertFalse(stats.isSuccessful()); - assertFileCountUnderDir(logsFolder, 2); - } - - private void assertFileCountUnderDir(File directory, int expectedFileCount) throws IOException { - String[] files = directory.list(); - assertNotNull(files); - assertEquals(expectedFileCount, files.length); - } - - private PigStats runPigLocally(String... extraArgs) { - String[] args = ArrayUtils.addAll(new String[]{"-x", "local"}, extraArgs); - return PigRunner.run(args, new TestNotificationListener("local")); - } - public static class TestNotificationListener implements PigProgressNotificationListener { private Map<String, int[]> numMap = new HashMap<String, int[]>();
Modified: pig/branches/spark/test/org/apache/pig/test/TestPigScriptParser.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPigScriptParser.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPigScriptParser.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPigScriptParser.java Fri Feb 24 03:34:37 2017 @@ -30,9 +30,6 @@ import java.util.Properties; import org.apache.pig.ExecType; import org.apache.pig.PigServer; -import org.apache.pig.builtin.mock.Storage; -import org.apache.pig.builtin.mock.Storage.Data; -import static org.apache.pig.builtin.mock.Storage.tuple; import org.apache.pig.data.Tuple; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.logicalLayer.FrontendException; @@ -102,7 +99,7 @@ public class TestPigScriptParser { @Test public void testDefineUDF() throws Exception { - PigServer ps = new PigServer(Util.getLocalTestMode()); + PigServer ps = new PigServer(ExecType.LOCAL); String inputData[] = { "dshfdskfwww.xyz.com/sportsjoadfjdslpdshfdskfwww.xyz.com/sportsjoadfjdsl" , "kas;dka;sd" , @@ -159,79 +156,6 @@ public class TestPigScriptParser { } } - @Test - public void testBackSlashOnly() throws Exception { - PigServer pig = new PigServer(Util.getLocalTestMode()); - Data data = Storage.resetData(pig); - data.set("input", tuple("abc"), tuple("\\bcd"), tuple("'cde"), tuple("def\\\\")); - - String query = - "A = load 'input' USING mock.Storage() as (a0:chararray);\n" - // java String is escaping "\" so the following line is equivalent of - // B = FILTER A by STARTSWITH(a0,'\\'); in the pig script - + "B = FILTER A by STARTSWITH(a0,'\\\\');\n" - + "store B into 'out' using mock.Storage;" ; - - Util.registerMultiLineQuery(pig, query); - List<Tuple> list = data.get("out"); - - assertEquals("There should be only one match", 1, list.size()); - Tuple t = list.get(0); - assertEquals("result should have only one field", 1, t.size() ); - assertEquals("\\bcd",(String) t.get(0)); - } - - - @Test - public void testBackSlashSingleQuote() throws Exception { - PigServer pig = new PigServer(Util.getLocalTestMode()); - Data data = Storage.resetData(pig); - data.set("input", tuple("abc"), tuple("\\bcd"), tuple("'cde"), tuple("def\\\\")); - - String query = - "A = load 'input' USING mock.Storage() as (a0:chararray);\n" - // java String is escaping "\" so the following line is equivalent of - // B = FILTER A by STARTSWITH(a0,'\''); in the pig script - + "B = FILTER A by STARTSWITH(a0,'\\'');\n" - + "store B into 'out' using mock.Storage;" ; - - Util.registerMultiLineQuery(pig, query); - List<Tuple> list = data.get("out"); - - assertEquals("There should be only one match", 1, list.size()); - Tuple t = list.get(0); - assertEquals("result should have only one field", 1, t.size() ); - assertEquals("'cde",(String) t.get(0)); - } - - @Test - public void testBackSlashReplace() throws Exception { - PigServer pig = new PigServer(Util.getLocalTestMode()); - Data data = Storage.resetData(pig); - //After java escaping, these tuples have - //'abc', '\bcd' and 'def\\' respectively - data.set("input", tuple("abc"), tuple("\\bcd"), tuple("def\\\\")); - - String query = - "A = load 'input' USING mock.Storage() as (a0:chararray);\n" - // java String is escaping "\" so the following line is equivalent of - //"B = FOREACH A GENERATE REPLACE(a0,'\\\\','+');\n" - + "B = FOREACH A GENERATE REPLACE(a0,'\\\\\\\\','+');\n" - + "store B into 'out' using mock.Storage;" ; - - // REPLACE(a0,'\\\\','+') - // --> Pig parser unescape and pass "\\" to REPLACE UDF. - // --> REPLACE UDF calls, Pattern.compile("\\"); which - // matches "\" - - Util.registerMultiLineQuery(pig, query); - List<Tuple> list = data.get("out"); - - List<Tuple> expectedRes = - Util.getTuplesFromConstantTupleStrings( - new String[] {"('abc')","('+bcd')", "('def++')"}); - Util.checkQueryOutputsAfterSort(list, expectedRes); - } private void checkParsedConstContent(PigServer pigServer, PigContext pigContext, String query, Modified: pig/branches/spark/test/org/apache/pig/test/TestPigStatsMR.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPigStatsMR.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPigStatsMR.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPigStatsMR.java Fri Feb 24 03:34:37 2017 @@ -103,7 +103,11 @@ public class TestPigStatsMR extends Test private static MROperPlan getMRPlan(PhysicalPlan pp, PigContext ctx) throws Exception { MapReduceLauncher launcher = new MapReduceLauncher(); - return launcher.compile(pp,ctx); + java.lang.reflect.Method compile = launcher.getClass() + .getDeclaredMethod("compile", + new Class[] { PhysicalPlan.class, PigContext.class }); + compile.setAccessible(true); + return (MROperPlan) compile.invoke(launcher, new Object[] { pp, ctx }); } private static String getAlias(MapReduceOper mro) throws Exception { Modified: pig/branches/spark/test/org/apache/pig/test/TestPlanGeneration.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPlanGeneration.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPlanGeneration.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPlanGeneration.java Fri Feb 24 03:34:37 2017 @@ -20,9 +20,6 @@ import static org.junit.Assert.assertNot import static org.junit.Assert.assertNull; import java.io.IOException; -import java.util.List; - -import junit.framework.Assert; import org.apache.hadoop.mapreduce.Job; import org.apache.pig.ExecType; @@ -39,22 +36,15 @@ import org.apache.pig.backend.hadoop.exe import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore; import org.apache.pig.builtin.PigStorage; -import org.apache.pig.builtin.mock.Storage; -import org.apache.pig.builtin.mock.Storage.Data; -import static org.apache.pig.builtin.mock.Storage.*; import org.apache.pig.data.DataType; -import org.apache.pig.data.Tuple; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.Utils; import org.apache.pig.newplan.Operator; -import org.apache.pig.newplan.logical.expression.CastExpression; import org.apache.pig.newplan.logical.expression.LogicalExpression; -import org.apache.pig.newplan.logical.expression.ProjectExpression; import org.apache.pig.newplan.logical.relational.LOCogroup; import org.apache.pig.newplan.logical.relational.LOFilter; import org.apache.pig.newplan.logical.relational.LOForEach; -import org.apache.pig.newplan.logical.relational.LOGenerate; import org.apache.pig.newplan.logical.relational.LOLoad; import org.apache.pig.newplan.logical.relational.LOSort; import org.apache.pig.newplan.logical.relational.LOStore; @@ -71,8 +61,8 @@ public class TestPlanGeneration { private static PigServer ps; @BeforeClass - public static void setUp() throws Exception { - ps = new PigServer(Util.getLocalTestMode()); + public static void setUp() throws ExecException { + ps = new PigServer(ExecType.LOCAL); pc = ps.getPigContext(); pc.connect(); } @@ -321,218 +311,4 @@ public class TestPlanGeneration { assertNotNull(((PartitionedLoader)loLoad.getLoadFunc()).getPartFilter()); assertEquals("b", loStore.getAlias()); } - - @Test - // See PIG-2315 - public void testForEachWithCast1() throws Exception { - // A cast ForEach is inserted to take care of the user schema - String query = "A = load 'foo' as (a, b:int);\n" + - "B = foreach A generate a as a0:chararray, b as b:int;\n" + - "store B into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach1 = (LOForEach)lp.getSuccessors(loLoad).get(0); - LOForEach loForEach2 = (LOForEach)lp.getSuccessors(loForEach1).get(0); - // before a0 is typecasted to chararray, it should be bytearray - assertEquals(DataType.BYTEARRAY, loForEach1.getSchema().getField(0).type); - // type of b should stay as int - assertEquals(DataType.INTEGER, loForEach1.getSchema().getField(1).type); - assertEquals("B", loForEach2.getAlias()); - LOGenerate generate = (LOGenerate)loForEach2.getInnerPlan().getSinks().get(0); - CastExpression cast = (CastExpression)generate.getOutputPlans().get(0).getSources().get(0); - Assert.assertTrue(cast.getType()==DataType.CHARARRAY); - assertEquals(loForEach2.getSchema().getField(0).alias, "a0"); - Assert.assertTrue(lp.getSuccessors(loForEach2).get(0) instanceof LOStore); - } - - @Test - // See PIG-2315 - public void testForEachWithCast2() throws Exception { - // No additional cast ForEach will be inserted, but schema should match - String query = "A = load 'foo' as (a, b);\n" + - "B = foreach A generate (chararray)a as a0:chararray;\n" + - "store B into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach = (LOForEach)lp.getSuccessors(loLoad).get(0); - assertEquals(loForEach.getSchema().getField(0).alias, "a0"); - Assert.assertTrue(lp.getSuccessors(loForEach).get(0) instanceof LOStore); - } - - @Test - // See PIG-2315 - public void testForEachWithCast3() throws Exception { - // No additional cast ForEach will be inserted, but schema should match - String query = "A = load 'foo' as (a, b);\n" + - "B = foreach A generate (chararray)a as a0:int;\n" + - "store B into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach1 = (LOForEach)lp.getSuccessors(loLoad).get(0); - LOGenerate generate1 = (LOGenerate)loForEach1.getInnerPlan().getSinks().get(0); - CastExpression cast1 = (CastExpression)generate1.getOutputPlans().get(0).getSources().get(0); - Assert.assertTrue(cast1.getType()==DataType.CHARARRAY); - //before a0 is typecasted to int, it should be chararray - Assert.assertEquals(DataType.CHARARRAY, loForEach1.getSchema().getField(0).type); - LOForEach loForEach2 = (LOForEach)lp.getSuccessors(loForEach1).get(0); - LOGenerate generate2 = (LOGenerate)loForEach2.getInnerPlan().getSinks().get(0); - CastExpression cast2 = (CastExpression)generate2.getOutputPlans().get(0).getSources().get(0); - Assert.assertTrue(cast2.getType()==DataType.INTEGER); - Assert.assertTrue(lp.getSuccessors(loForEach2).get(0) instanceof LOStore); - } - - @Test - // See PIG-2315 - public void testForEachWithCast4() throws Exception { - // No additional cast ForEach will be inserted - String query = "a = load 'foo' as (nb1:bag{}, nb2:chararray);\n" + - "b = foreach a generate flatten(nb1) as (year, name), nb2;\n" + - "store b into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach = (LOForEach)lp.getSuccessors(loLoad).get(0); - Assert.assertTrue(lp.getSuccessors(loForEach).get(0) instanceof LOStore); - } - - @Test - // See PIG-2315 - public void testForEachWithCast5() throws Exception { - // cast ForEach will be inserted - String query = "a = load 'foo' as (nb1:bag{}, nb2:chararray);\n" + - "b = foreach a generate flatten(nb1) as (year, name:chararray), nb2 as nb2:chararray;\n" + - "store b into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach1 = (LOForEach)lp.getSuccessors(loLoad).get(0); - // flattened "name" field should be bytearray before typecasted to chararray - Assert.assertEquals(DataType.BYTEARRAY, loForEach1.getSchema().getField(1).type); - LOForEach loForEach2 = (LOForEach)lp.getSuccessors(loForEach1).get(0); - LOGenerate generate = (LOGenerate)loForEach2.getInnerPlan().getSinks().get(0); - Assert.assertTrue(generate.getOutputPlans().get(0).getSources().get(0) instanceof ProjectExpression); - CastExpression cast = (CastExpression)generate.getOutputPlans().get(1).getSources().get(0); - Assert.assertTrue(cast.getType()==DataType.CHARARRAY); - Assert.assertTrue(generate.getOutputPlans().get(2).getSources().get(0) instanceof ProjectExpression); - } - - @Test - // See PIG-2315 - public void testForEachWithCast6() throws Exception { - // no cast ForEach will be inserted - String query = "a = load 'foo' as (nb1:bag{(year,name)}, nb2);\n" + - "b = foreach a generate flatten(nb1) as (year, name2), nb2;\n" + - "store b into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach = (LOForEach)lp.getSuccessors(loLoad).get(0); - assertEquals(loForEach.getSchema().getField(1).alias, "name2"); - Assert.assertTrue(lp.getSuccessors(loForEach).get(0) instanceof LOStore); - } - - @Test - // See PIG-2315 - public void testForEachWithCast7() throws Exception { - // no cast ForEach will be inserted, since we don't know the size of outputs - // in first inner plan - String query = "a = load 'foo' as (nb1:bag{}, nb2:bag{});\n" + - "b = foreach a generate flatten(nb1), flatten(nb2) as (year, name);\n" + - "store b into 'output';"; - - LogicalPlan lp = Util.parse(query, pc); - Util.optimizeNewLP(lp); - - LOLoad loLoad = (LOLoad)lp.getSources().get(0); - LOForEach loForEach = (LOForEach)lp.getSuccessors(loLoad).get(0); - Assert.assertTrue(lp.getSuccessors(loForEach).get(0) instanceof LOStore); - } - - @Test - // See PIG-2315 - public void testAsType1() throws Exception { - Data data = Storage.resetData(ps); - data.set("input", tuple(0.1), tuple(1.2), tuple(2.3)); - - String query = - "A = load 'input' USING mock.Storage() as (a1:double);\n" - + "B = FOREACH A GENERATE a1 as (a2:int);\n" - + "store B into 'out' using mock.Storage;" ; - - Util.registerMultiLineQuery(ps, query); - List<Tuple> list = data.get("out"); - // Without PIG-2315, this failed with (0.1), (1.2), (2.3) - List<Tuple> expectedRes = - Util.getTuplesFromConstantTupleStrings( - new String[] {"(0)", "(1)", "(2)"}); - Util.checkQueryOutputsAfterSort(list, expectedRes); - } - - @Test - // See PIG-2315 - public void testAsType2() throws Exception { - Data data = Storage.resetData(ps); - data.set("input", tuple("a"), tuple("b"), tuple("c")); - - String query = - "A = load 'input' USING mock.Storage(); \n" - + "A2 = FOREACH A GENERATE 12345 as (a2:chararray); \n" - + "B = load 'input' USING mock.Storage(); \n" - + "B2 = FOREACH A GENERATE '12345' as (b2:chararray); \n" - + "C = union A2, B2;\n" - + "D = distinct C;\n" - + "store D into 'out' using mock.Storage;" ; - - Util.registerMultiLineQuery(ps, query); - List<Tuple> list = data.get("out"); - // Without PIG-2315, this produced TWO 12345. - // One by chararray and another by int. - List<Tuple> expectedRes = - Util.getTuplesFromConstantTupleStrings( - new String[] {"('12345')"}); - Util.checkQueryOutputsAfterSort(list, expectedRes); - } - - @Test - // See PIG-4933 - public void testAsWithByteArrayCast() throws Exception { - Data data = Storage.resetData(ps); - data.set("input_testAsWithByteArrayCast", "t1:(f1:bytearray, f2:bytearray), f3:chararray", - tuple(tuple(1,5), "a"), - tuple(tuple(2,4), "b"), - tuple(tuple(3,3), "c") ); - - String query = - "A = load 'input_testAsWithByteArrayCast' USING mock.Storage();\n" - + "B = FOREACH A GENERATE t1 as (t2:(newf1, newf2:float)), f3;" - + "store B into 'out' using mock.Storage;" ; - - // This will call typecast of (bytearray,float) on a tuple - // bytearray2bytearray should be no-op. - // Without pig-4933 patch on POCast, - // this typecast was producing empty results - - Util.registerMultiLineQuery(ps, query); - List<Tuple> list = data.get("out"); - String[] expectedRes = - new String[] {"((1,5.0),a)","((2,4.0),b)","((3,3.0),c)"}; - for( int i=0; i < list.size(); i++ ) { - Assert.assertEquals(expectedRes[i], list.get(i).toString()); - } - } } Modified: pig/branches/spark/test/org/apache/pig/test/TestPruneColumn.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPruneColumn.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPruneColumn.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPruneColumn.java Fri Feb 24 03:34:37 2017 @@ -1397,13 +1397,13 @@ public class TestPruneColumn { } @Test - public void testFlattenMapCantPruneKeys() throws Exception { + public void testRelayFlattenMap() throws Exception { pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile3.toString(), pigServer.getPigContext()) - + "' as (a0, a1:map[int]);"); + + "' as (a0, a1:map[]);"); pigServer.registerQuery("B = foreach A generate flatten(a1);"); - pigServer.registerQuery("B1 = filter B by a1::key == 'key1';"); - pigServer.registerQuery("C = foreach B1 generate a1::value;"); + pigServer.registerQuery("C = foreach B generate a1#'key1';"); + Iterator<Tuple> iter = pigServer.openIterator("C"); assertTrue(iter.hasNext()); @@ -1418,7 +1418,8 @@ public class TestPruneColumn { assertFalse(iter.hasNext()); - assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0"})); + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0", + "Map key required for A: $1->[key1]"})); } @Test Modified: pig/branches/spark/test/org/apache/pig/test/TestRegisterParser.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestRegisterParser.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestRegisterParser.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestRegisterParser.java Fri Feb 24 03:34:37 2017 @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.Writer; import java.net.URI; import java.net.URISyntaxException; -import java.util.Properties; import org.apache.pig.ExecType; import org.apache.pig.impl.PigContext; @@ -43,9 +42,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import org.apache.pig.impl.util.PropertiesUtil; -import org.apache.hadoop.fs.LocalFileSystem; - public class TestRegisterParser { private PigServer pigServer; @@ -53,12 +49,7 @@ public class TestRegisterParser { @Before public void setUp() throws Exception { - Properties properties = PropertiesUtil.loadDefaultProperties(); - properties.setProperty("fs.s3.impl", LocalFileSystem.class.getName()); - properties.setProperty("fs.s3n.impl", LocalFileSystem.class.getName()); - properties.setProperty("fs.s3a.impl", LocalFileSystem.class.getName()); - - pigServer = new PigServer(ExecType.LOCAL, properties); + pigServer = new PigServer(ExecType.LOCAL); // Generate test jar files for (int i = 1; i <= 5; i++) { @@ -116,34 +107,6 @@ public class TestRegisterParser { } } - @Test - public void testResolveForVariousFileSystemSchemes() throws URISyntaxException, IOException, ParserException { - URI[] list = new URI[6]; - list[0] = new URI("file://test.jar"); - list[1] = new URI("hdfs://test.jar"); - list[2] = new URI("s3://test.jar"); - list[3] = new URI("s3n://test.jar"); - list[4] = new URI("s3a://test.jar"); - list[5] = new URI("test.jar"); - - RegisterResolver registerResolver = new RegisterResolver(pigServer); - for (URI uri : list) { - URI[] resolvedUris = registerResolver.resolve(uri); - Assert.assertEquals(1, resolvedUris.length); - Assert.assertEquals(uri, resolvedUris[0]); - } - } - - @Test(expected = ParserException.class) - public void testResolveParseException() throws URISyntaxException, IOException, ParserException { - new RegisterResolver(pigServer).resolve(new URI("abc://test.jar")); - } - - @Test(expected = URISyntaxException.class) - public void testResolveURISyntaxException() throws URISyntaxException, IOException, ParserException { - new RegisterResolver(pigServer).resolve(new URI("123://test.jar")); - } - // Throw error when a scripting language and namespace is specified for a jar @Test(expected = ParserException.class) public void testRegisterJarException1() throws IOException, ParserException { Modified: pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java Fri Feb 24 03:34:37 2017 @@ -247,11 +247,7 @@ public class TestScriptUDF{ Assert.assertTrue(t.get(0).toString().equals(System.getenv(input[0]))); Assert.assertTrue(iter.hasNext()); t = iter.next(); - if (System.getenv(input[1]) != null) { // JAVA_HOME is set, t.get(0) is not null - Assert.assertTrue(t.get(0).toString().equals(System.getenv(input[1]))); - } else { // JAVA_HOME is not set, t.get(0) is null - Assert.assertNull(t.get(0)); - } + Assert.assertTrue(t.get(0).toString().equals(System.getenv(input[1]))); Assert.assertFalse(iter.hasNext()); } Modified: pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java Fri Feb 24 03:34:37 2017 @@ -65,7 +65,6 @@ public class TestSkewedJoin { private static final String INPUT_FILE5 = "SkewedJoinInput5.txt"; private static final String INPUT_FILE6 = "SkewedJoinInput6.txt"; private static final String INPUT_FILE7 = "SkewedJoinInput7.txt"; - private static final String INPUT_FILE8 = "SkewedJoinInput8.txt"; private static final String TEST_DIR = Util.getTestDirectory(TestSkewedJoin.class); private static final String INPUT_DIR = TEST_DIR + Path.SEPARATOR + "input"; private static final String OUTPUT_DIR = TEST_DIR + Path.SEPARATOR + "output"; @@ -174,11 +173,6 @@ public class TestSkewedJoin { } w7.close(); - //Empty file - PrintWriter w8 = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE8)); - w8.close(); - - Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE1, INPUT_FILE1); Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE2, INPUT_FILE2); Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE3, INPUT_FILE3); @@ -186,7 +180,6 @@ public class TestSkewedJoin { Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE5, INPUT_FILE5); Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE6, INPUT_FILE6); Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE7, INPUT_FILE7); - Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE8, INPUT_FILE8); } private static void deleteFiles() throws IOException { @@ -194,21 +187,6 @@ public class TestSkewedJoin { } @Test - public void testSkewedJoinMapLeftEmpty() throws IOException{ - pigServer.registerQuery("A = LOAD '" + INPUT_FILE8 + "' as (idM:[]);"); - pigServer.registerQuery("B = LOAD '" + INPUT_FILE1 + "' as (id, name, n);"); - pigServer.registerQuery("C = join A by idM#'id', B by id using 'skewed' PARALLEL 2;"); - Iterator<Tuple> iter = pigServer.openIterator("C"); - int count = 0; - while(iter.hasNext()) { - count++; - iter.next(); - } - assertEquals(0, count); - } - - - @Test public void testSkewedJoinWithGroup() throws IOException{ pigServer.registerQuery("A = LOAD '" + INPUT_FILE1 + "' as (id, name, n);"); pigServer.registerQuery("B = LOAD '" + INPUT_FILE2 + "' as (id, name);"); Modified: pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java Fri Feb 24 03:34:37 2017 @@ -18,7 +18,6 @@ package org.apache.pig.test; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; @@ -373,41 +372,4 @@ public class TestStreamingLocal { Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults); } } - - @Test - // Perl script with a syntax error, See PIG-4976 - public void testNegativeScriptSyntaxError() throws IOException { - - for( int numinput : new int [] {10, 9999} ) { - String[] inputStrings = new String[numinput]; - for (int i=0;i<numinput;i++) { - inputStrings[i] = Integer.toString(i); - } - File input = Util.createInputFile("tmp", "", inputStrings); - // Perl script - String[] script = - new String[] { - "#!/usr/bin/perl", - "syntax error", - }; - File command1 = Util.createInputFile("script", "pl", script); - String query = - "define CMD `perl " + command1.getName() + "` output('foo')" + - "ship ('" + Util.encodeEscape(command1.toString()) + "');"; - boolean succeeded=true; - try { - pigServer.registerQuery( query ); - pigServer.registerQuery("A = load '" - + Util.generateURI(input.toString(), - pigServer.getPigContext()) - + "' using PigStorage();"); - pigServer.registerQuery("B = stream A through CMD;"); - pigServer.openIterator("B"); - } catch(Exception ex) { - succeeded=false; - } - Assert.assertFalse("Job with " + numinput + " lines input did not fail.", succeeded); - } - } - } Modified: pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Fri Feb 24 03:34:37 2017 @@ -39,10 +39,8 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Properties; import junit.framework.Assert; @@ -57,8 +55,6 @@ import org.apache.pig.builtin.PigStorage import org.apache.pig.data.DataBag; import org.apache.pig.data.DataByteArray; import org.apache.pig.data.DataType; -import org.apache.pig.data.DefaultTuple; -import org.apache.pig.data.NonSpillableDataBag; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.impl.PigContext; @@ -2912,12 +2908,12 @@ public class TestTypeCheckingValidatorNe @Test public void testUnionLineageDifferentSchemaFail() throws Throwable { - String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );\n" - + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray, field7 );\n" - + "c = union a , b;\n" + String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );" + + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray, field7 );" + + "c = union a , b;" + "d = foreach c generate $3 + 2.0 ;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to double at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } private void checkWarning(String query, String warnMsg) throws FrontendException { @@ -2959,12 +2955,12 @@ public class TestTypeCheckingValidatorNe public void testUnionLineageMixSchemaFail() throws Throwable { // different loader caster associated with each input, so can't determine // which one to use on union output - String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );\n" - + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');\n" - + "c = union a , b;\n" + String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );" + + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');" + + "c = union a , b;" + "d = foreach c generate $3 + 2.0 ;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to double at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } @Test @@ -3306,12 +3302,12 @@ public class TestTypeCheckingValidatorNe @Test public void testCrossLineageNoSchemaFail() throws Throwable { - String query = "a = load 'a' using PigStorage('a');\n" - + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');\n" - + "c = cross a , b;\n" + String query = "a = load 'a' using PigStorage('a');" + + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');" + + "c = cross a , b;" + "d = foreach c generate $1 + 2.0 ;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to double at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } @Test @@ -3327,12 +3323,12 @@ public class TestTypeCheckingValidatorNe @Test public void testCrossLineageMixSchemaFail() throws Throwable { - String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );\n" - + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');\n" - + "c = cross a , b;\n" + String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );" + + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');" + + "c = cross a , b;" + "d = foreach c generate $3 + 2.0 ;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to double at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } @Test @@ -3361,12 +3357,12 @@ public class TestTypeCheckingValidatorNe public void testJoinLineageNoSchemaFail() throws Throwable { //this test case should change when we decide on what flattening a tuple or bag //with null schema results in a foreach flatten and hence a join - String query = "a = load 'a' using PigStorage('a');\n" - + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();\n" - + "c = join a by $0, b by $0;\n" + String query = "a = load 'a' using PigStorage('a');" + + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();" + + "c = join a by $0, b by $0;" + "d = foreach c generate $1 + 2.0 ;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to double at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } @Test @@ -3382,12 +3378,12 @@ public class TestTypeCheckingValidatorNe public void testJoinLineageMixSchemaFail() throws Throwable { //this test case should change when we decide on what flattening a tuple or bag //with null schema results in a foreach flatten and hence a join - String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );\n" - + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();\n" - + "c = join a by field1, b by $0;\n" + String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );" + + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();" + + "c = join a by field1, b by $0;" + "d = foreach c generate $3 + 2.0 ;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to double at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } @Test @@ -3871,12 +3867,12 @@ public class TestTypeCheckingValidatorNe */ @Test public void testLineageMultipleLoader3() throws FrontendException { - String query = "A = LOAD 'data1' USING PigStorage() AS (u, v, w);\n" - + "B = LOAD 'data2' USING TextLoader() AS (x, y);\n" - + "C = COGROUP A BY u, B by x;\n" - + "D = FOREACH C GENERATE (chararray)group;\n"; + String query = "A = LOAD 'data1' USING PigStorage() AS (u, v, w);" + + "B = LOAD 'data2' USING TextLoader() AS (x, y);" + + "C = COGROUP A BY u, B by x;" + + "D = FOREACH C GENERATE (chararray)group;"; - checkWarning(query, CAST_LOAD_NOT_FOUND + " to chararray at <line 4,"); + checkWarning(query, CAST_LOAD_NOT_FOUND); } /** @@ -4067,12 +4063,12 @@ public class TestTypeCheckingValidatorNe @Test public void testUDFNoInnerSchema() throws FrontendException { - String query = "a= load '1.txt' using PigStorage(':') ;" + String query = "a= load '1.txt';" + "b = foreach a generate "+TestUDFTupleNullInnerSchema.class.getName()+"($0);" + "c = foreach b generate flatten($0);" + "d = foreach c generate $0 + 1;"; - checkLastForeachCastLoadFunc(query, "PigStorage(':')"); + checkLastForeachCastLoadFunc(query, null, 0); } //see PIG-1990 @@ -4122,56 +4118,4 @@ public class TestTypeCheckingValidatorNe " corresponding column in earlier relation(s) in the statement"; Util.checkExceptionMessage(query, "c", msg); } - //see PIG-4734 - public static class GenericToMap extends EvalFunc<Map<String, Double>> { - @Override - public Map exec(Tuple input) throws IOException { - Map<String, Double> output = new HashMap<String, Double>(); - output.put((String)input.get(0), (Double)input.get(1)); - return output; - } - } - @Test - public void testBinCondCompatMap() throws Exception { - String query = - "a = load 'studenttab10k' as (name:chararray, gpa:double);" - + "b = foreach a generate gpa, TOMAP(name, gpa) as m1, " - + GenericToMap.class.getName() + "(name, gpa) as m2;" - + "c = foreach b generate (gpa>3? m1 : m2);"; - createAndProcessLPlan(query); - } - public static class GenericToTuple extends EvalFunc<Tuple> { - @Override - public Tuple exec(Tuple input) throws IOException { - return input; - } - } - @Test - public void testBinCondCompatTuple() throws Exception { - String query = - "a = load 'studenttab10k' as (name:chararray, gpa:double);" - + "b = foreach a generate gpa, TOTUPLE(name, gpa) as t1, " - + GenericToTuple.class.getName() + "(name, gpa) as t2;" - + "c = foreach b generate (gpa>3? t1 : t2);"; - createAndProcessLPlan(query); - } - public static class GenericToBag extends EvalFunc<DataBag> { - @Override - public DataBag exec(Tuple input) throws IOException { - DataBag bag = new NonSpillableDataBag(1); - Tuple t = new DefaultTuple(); - t.append(input.get(0)); - bag.add(t); - return bag; - } - } - @Test - public void testBinCondCompatBag() throws Exception { - String query = - "a = load 'studenttab10k' as (name:chararray, gpa:double);" - + "b = foreach a generate gpa, TOBAG(name) as b1, " - + GenericToBag.class.getName() + "(name) as b2;" - + "c = foreach b generate (gpa>3? b1 : b2);"; - createAndProcessLPlan(query); - } } Modified: pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java Fri Feb 24 03:34:37 2017 @@ -96,6 +96,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA on two inputs with same schema + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaSameSchema() throws Exception { @@ -126,6 +128,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with operations after the union + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaFilter() throws Exception { @@ -157,6 +161,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with operations after the union + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaSuccOps() throws Exception { @@ -188,6 +194,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with cast from bytearray to another type + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaCastOnByteArray() throws Exception { @@ -215,6 +223,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA where a common column has additional 'namespace' part * in the column name in one of the inputs + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaScopedColumnName() throws Exception { @@ -256,6 +266,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA where a common column has additional 'namespace' part * in the column name in both the inputs + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaScopedColumnNameBothInp1() throws Exception { @@ -290,6 +302,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA where a common column has additional 'namespace' part * in the column name in both the inputs + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaScopedColumnNameBothInp2() throws Exception { @@ -326,6 +340,8 @@ public class TestUnionOnSchema { * Test UNION ONSCHEMA where a common column has additional 'namespace' part * in the column name in one of the inputs. * Negative test case + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaScopedColumnNameNeg() throws Exception { @@ -350,6 +366,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA on two inputs with same column names, but different * numeric types - test type promotion + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaDiffNumType() throws Exception { @@ -378,6 +396,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA on two inputs with no common columns + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaNoCommonCols() throws Exception { @@ -404,6 +424,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA on two inputs , one input with additional columns + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaAdditionalColumn() throws Exception { @@ -476,6 +498,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA on 3 inputs + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchema3Inputs() throws Exception { @@ -509,6 +533,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with bytearray type + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaByteArrayConversions() throws Exception { @@ -546,6 +572,8 @@ public class TestUnionOnSchema { /** * negative test - test error on no schema + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaNoSchema() throws Exception { @@ -569,6 +597,8 @@ public class TestUnionOnSchema { /** * negative test - test error on null alias in one of the FieldSchema + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaNullAliasInFieldSchema() throws Exception { @@ -610,6 +640,8 @@ public class TestUnionOnSchema { /** * test union with incompatible types in schema + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaIncompatibleTypes() throws Exception { @@ -618,15 +650,7 @@ public class TestUnionOnSchema { + "l2 = load '" + INP_FILE_2NUMS + "' as (x : long, y : float);" + "u = union onschema l1, l2;"; - checkSchemaEx(query, "Cannot cast from chararray to bytearray"); - - //without "onschema" - query = - " l1 = load '" + INP_FILE_2NUMS + "' as (x : long, y : chararray);" - + "l2 = load '" + INP_FILE_2NUMS + "' as (x : long, y : float);" - + "u = union l1, l2;"; - - checkSchemaEx(query, "Cannot cast from chararray to bytearray"); + checkSchemaEquals(query, "x : long, y : bytearray"); @@ -635,15 +659,8 @@ public class TestUnionOnSchema { + "l2 = load '" + INP_FILE_2NUMS + "' as (x : map[ ], y : chararray);" + "u = union onschema l1, l2;" ; - checkSchemaEx(query, "Cannot cast from long to bytearray"); + checkSchemaEquals(query, "x : bytearray, y : chararray"); - query = - " l1 = load '" + INP_FILE_2NUMS + "' as (x : long, y : chararray);" - + "l2 = load '" + INP_FILE_2NUMS + "' as (x : map[ ], y : chararray);" - + "u = union l1, l2;" - ; - checkSchemaEx(query, "Cannot cast from long to bytearray"); - // bag column with different internal column types query = " l1 = load '" + INP_FILE_2NUMS @@ -691,6 +708,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with input relation having udfs + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaInputUdfs() throws Exception { @@ -726,6 +745,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with udf whose default type is different from * final type + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaUdfTypeEvolution() throws Exception { @@ -776,6 +797,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with udf whose default type is different from * final type - where udf is not in immediate input of union + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaUdfTypeEvolution2() throws Exception { @@ -846,6 +869,8 @@ public class TestUnionOnSchema { /** * Test UNION ONSCHEMA with input relation having column names with multiple * level of namespace in their names + * @throws IOException + * @throws ParserException */ @Test public void testUnionOnSchemaScopeMulti() throws Exception { @@ -891,6 +916,8 @@ public class TestUnionOnSchema { /** * Test query with a union-onschema having another as input + * @throws IOException + * @throws ParserException */ @Test public void testTwoUnions() throws Exception { Modified: pig/branches/spark/test/org/apache/pig/test/Util.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/Util.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/Util.java (original) +++ pig/branches/spark/test/org/apache/pig/test/Util.java Fri Feb 24 03:34:37 2017 @@ -480,19 +480,6 @@ public class Util { fs.delete(new Path(fileName), true); } - /** - * Deletes a dfs file from the MiniCluster DFS quietly - * - * @param miniCluster the MiniCluster where the file should be deleted - * @param fileName the path of the file to be deleted - */ - public static void deleteQuietly(MiniGenericCluster miniCluster, String fileName) { - try { - deleteFile(miniCluster, fileName); - } catch (IOException ignored) { - } - } - static public void deleteFile(PigContext pigContext, String fileName) throws IOException { Configuration conf = ConfigurationUtil.toConfiguration( @@ -671,10 +658,13 @@ public class Util { } } - static private String getFSMkDirCommand(String fileName) { - Path parentDir = new Path(fileName).getParent(); - String mkdirCommand = parentDir.getName().isEmpty() ? "" : "fs -mkdir -p " + parentDir + "\n"; - return mkdirCommand; + static private String getMkDirCommandForHadoop2_0(String fileName) { + if (org.apache.pig.impl.util.Utils.isHadoop23() || org.apache.pig.impl.util.Utils.isHadoop2()) { + Path parentDir = new Path(fileName).getParent(); + String mkdirCommand = parentDir.getName().isEmpty() ? "" : "fs -mkdir -p " + parentDir + "\n"; + return mkdirCommand; + } + return ""; } /** @@ -696,7 +686,7 @@ public class Util { fileNameOnCluster = fileNameOnCluster.replace('\\','/'); } PigServer ps = new PigServer(cluster.getExecType(), cluster.getProperties()); - String script = getFSMkDirCommand(fileNameOnCluster) + "fs -put " + localFileName + " " + fileNameOnCluster; + String script = getMkDirCommandForHadoop2_0(fileNameOnCluster) + "fs -put " + localFileName + " " + fileNameOnCluster; GruntParser parser = new GruntParser(new StringReader(script), ps); parser.setInteractive(false); try { @@ -857,23 +847,7 @@ public class Util { } public static File createFile(String[] data) throws Exception{ - return createFile(null,data); - } - - public static File createFile(String filePath, String[] data) throws Exception { - File f; - if( null == filePath || filePath.isEmpty() ) { - f = File.createTempFile("tmp", ""); - } else { - f = new File(filePath); - } - - if (f.getParent() != null && !(new File(f.getParent())).exists()) { - (new File(f.getParent())).mkdirs(); - } - - f.deleteOnExit(); - + File f = File.createTempFile("tmp", ""); PrintWriter pw = new PrintWriter(f); for (int i=0; i<data.length; i++){ pw.println(data[i]); @@ -944,7 +918,14 @@ public class Util { MapRedUtil.checkLeafIsStore(pp, pc); MapReduceLauncher launcher = new MapReduceLauncher(); - return launcher.compile(pp,pc); + + java.lang.reflect.Method compile = launcher.getClass() + .getDeclaredMethod("compile", + new Class[] { PhysicalPlan.class, PigContext.class }); + + compile.setAccessible(true); + + return (MROperPlan) compile.invoke(launcher, new Object[] { pp, pc }); } public static MROperPlan buildMRPlan(String query, PigContext pc) throws Exception { Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cogroup-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cogroup-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cogroup-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cogroup-1.gld Fri Feb 24 03:34:37 2017 @@ -44,7 +44,7 @@ c: Local Rearrange[tuple]{int}(false) - |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8 Tez vertex scope-36 # Plan on vertex -d: Store(file:///tmp/pigoutput/d:org.apache.pig.builtin.PigStorage) - scope-33 +d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33 | |---d: New For Each(false,false,false)[bag] - scope-32 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld Fri Feb 24 03:34:37 2017 @@ -64,7 +64,7 @@ c: Local Rearrange[tuple]{tuple}(false) |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8 Tez vertex scope-44 # Plan on vertex -d: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-41 +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-41 | |---d: New For Each(false,false,false)[bag] - scope-40 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld Fri Feb 24 03:34:37 2017 @@ -72,7 +72,7 @@ a: Split - scope-50 |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-49 # Plan on vertex -d: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-42 +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-42 | |---d: New For Each(true,true)[tuple] - scope-41 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-3.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-3.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-3.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-3.gld Fri Feb 24 03:34:37 2017 @@ -78,7 +78,7 @@ c: Local Rearrange[tuple]{tuple}(false) |---POValueInputTez - scope-59 <- scope-53 Tez vertex scope-60 # Plan on vertex -d: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-52 +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-52 | |---d: New For Each(false,false,false)[bag] - scope-51 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-1.gld Fri Feb 24 03:34:37 2017 @@ -26,16 +26,24 @@ b: Local Rearrange[tuple]{tuple}(true) - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-16 # Combine plan on edge <scope-13> -org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.DistinctCombiner$Combine +Local Rearrange[tuple]{tuple}(true) - scope-21 -> scope-16 +| | +| Project[tuple][*] - scope-20 +| +|---New For Each(true)[bag] - scope-19 + | | + | Project[tuple][0] - scope-18 + | + |---Package(Packager)[tuple]{tuple} - scope-17 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-12 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-12 | |---c: New For Each(false)[bag] - scope-11 | | | Project[int][1] - scope-9 | - |---New For Each(true)[bag] - scope-19 + |---New For Each(true)[bag] - scope-24 | | - | Project[tuple][0] - scope-18 + | Project[tuple][0] - scope-23 | - |---Package(Packager)[tuple]{tuple} - scope-17 + |---Package(Packager)[tuple]{tuple} - scope-22 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-2.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-2.gld Fri Feb 24 03:34:37 2017 @@ -50,7 +50,7 @@ b: Local Rearrange[tuple]{int}(false) - | |---b: Package(CombinerPackager)[tuple]{int} - scope-31 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-17 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-17 | |---c: New For Each(false)[bag] - scope-16 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-1.gld Fri Feb 24 03:34:37 2017 @@ -44,7 +44,7 @@ Local Rearrange[tuple]{int}(false) - sco |---c: Load(file:///tmp/input3:org.apache.pig.builtin.PigStorage) - scope-16 Tez vertex scope-36 # Plan on vertex -d: Store(file:///tmp/pigoutput/d:org.apache.pig.builtin.PigStorage) - scope-35 +d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-35 | |---d: FRJoin[tuple] - scope-27 <- scope-37,scope-38 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-2.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-2.gld Fri Feb 24 03:34:37 2017 @@ -70,7 +70,7 @@ b: Local Rearrange[tuple]{int}(false) - | |---b: Package(CombinerPackager)[tuple]{int} - scope-50 # Plan on vertex -d: Store(file:///tmp/pigoutput/e:org.apache.pig.builtin.PigStorage) - scope-35 +d: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-35 | |---d: FRJoin[tuple] - scope-29 <- scope-38 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Filter-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Filter-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Filter-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Filter-1.gld Fri Feb 24 03:34:37 2017 @@ -8,7 +8,7 @@ Tez vertex scope-16 Tez vertex scope-16 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-15 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-15 | |---c: New For Each(false)[bag] - scope-14 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Group-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Group-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Group-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Group-1.gld Fri Feb 24 03:34:37 2017 @@ -52,7 +52,7 @@ b: Local Rearrange[tuple]{int}(false) - | |---b: Package(CombinerPackager)[tuple]{int} - scope-33 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-19 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-19 | |---c: New For Each(false,false)[bag] - scope-18 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Join-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Join-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Join-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Join-1.gld Fri Feb 24 03:34:37 2017 @@ -44,7 +44,7 @@ c: Local Rearrange[tuple]{int}(false) - |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8 Tez vertex scope-37 # Plan on vertex -d: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-34 +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-34 | |---d: New For Each(false,false,false)[bag] - scope-33 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld Fri Feb 24 03:34:37 2017 @@ -16,7 +16,7 @@ POValueOutputTez - scope-11 -> [scope-1 |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-12 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-9 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-9 | |---c: New For Each(false)[bag] - scope-8 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld Fri Feb 24 03:34:37 2017 @@ -88,7 +88,7 @@ POValueOutputTez - scope-50 -> [scope-5 |---Package(LitePackager)[tuple]{tuple} - scope-46 Tez vertex scope-51 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-11 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-11 | |---Limit - scope-53 | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld Fri Feb 24 03:34:37 2017 @@ -134,7 +134,7 @@ POValueOutputTez - scope-66 -> [scope-6 |---Package(LitePackager)[tuple]{tuple} - scope-63 Tez vertex scope-67 # Plan on vertex -c: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-32 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-32 | |---c: Limit - scope-69 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld Fri Feb 24 03:34:37 2017 @@ -11,7 +11,7 @@ Tez vertex scope-18 Tez vertex scope-18 # Plan on vertex -a: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-8 +a: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-8 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -31,7 +31,7 @@ Tez vertex scope-19 Tez vertex scope-19 # Plan on vertex -b: Store(file:///tmp/pigoutput1:org.apache.pig.builtin.PigStorage) - scope-17 +b: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-17 | |---b: New For Each(false,false)[bag] - scope-16 | | @@ -43,4 +43,4 @@ b: Store(file:///tmp/pigoutput1:org.apac | | | |---Project[bytearray][1] - scope-13 | - |---b: Load(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-9 + |---b: Load(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-9 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld Fri Feb 24 03:34:37 2017 @@ -1,9 +1,11 @@ #-------------------------------------------------- -# There are 3 DAGs in the session +# There are 5 DAGs in the session #-------------------------------------------------- -Tez DAG pig-0_scope-0 -> Tez DAG pig-1_scope-1, -Tez DAG pig-1_scope-1 -> Tez DAG pig-2_scope-2, +Tez DAG pig-0_scope-0 -> Tez DAG pig-1_scope-1,Tez DAG pig-3_scope-3,Tez DAG pig-4_scope-4, +Tez DAG pig-1_scope-1 -> Tez DAG pig-2_scope-2,Tez DAG pig-4_scope-4, Tez DAG pig-2_scope-2 +Tez DAG pig-3_scope-3 -> Tez DAG pig-4_scope-4, +Tez DAG pig-4_scope-4 #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 @@ -12,62 +14,70 @@ Tez vertex scope-20 Tez vertex scope-20 # Plan on vertex -a: Store(file:///tmp/pigoutput/Dir1:org.apache.pig.builtin.PigStorage) - scope-1 +a: Store(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-1 | |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 #-------------------------------------------------- # TEZ DAG plan: pig-1_scope-1 #-------------------------------------------------- Tez vertex scope-21 -Tez vertex scope-23 Tez vertex scope-21 # Plan on vertex -a: Store(file:///tmp/pigoutput/Dir2:BinStorage) - scope-3 +a: Store(file:///tmp/output/Dir2:BinStorage) - scope-3 | -|---a: Load(file:///tmp/pigoutput/Dir1:org.apache.pig.builtin.PigStorage) - scope-2 -Tez vertex scope-23 -# Plan on vertex -a: Store(file:///tmp/pigoutput/Dir3:org.apache.pig.builtin.PigStorage) - scope-7 -| -|---a: Load(file:///tmp/pigoutput/Dir1:org.apache.pig.builtin.PigStorage) - scope-6 +|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-2 #-------------------------------------------------- # TEZ DAG plan: pig-2_scope-2 #-------------------------------------------------- Tez vertex scope-22 -Tez vertex scope-24 -> Tez vertex scope-27, -Tez vertex scope-25 -> Tez vertex scope-27, -Tez vertex scope-26 -> Tez vertex scope-27, -Tez vertex scope-27 Tez vertex scope-22 # Plan on vertex -a: Store(file:///tmp/pigoutput/Dir4:org.apache.pig.builtin.PigStorage) - scope-5 +a: Store(file:///tmp/output/Dir4:org.apache.pig.builtin.PigStorage) - scope-5 | -|---a: Load(file:///tmp/pigoutput/Dir2:BinStorage) - scope-4 -Tez vertex scope-24 +|---a: Load(file:///tmp/output/Dir2:BinStorage) - scope-4 +#-------------------------------------------------- +# TEZ DAG plan: pig-3_scope-3 +#-------------------------------------------------- +Tez vertex scope-23 + +Tez vertex scope-23 # Plan on vertex -d: Local Rearrange[tuple]{bytearray}(false) - scope-13 -> scope-27 +a: Store(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-7 +| +|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-6 +#-------------------------------------------------- +# TEZ DAG plan: pig-4_scope-4 +#-------------------------------------------------- +Tez vertex scope-26 -> Tez vertex scope-27, +Tez vertex scope-25 -> Tez vertex scope-27, +Tez vertex scope-24 -> Tez vertex scope-27, +Tez vertex scope-27 + +Tez vertex scope-26 +# Plan on vertex +d: Local Rearrange[tuple]{bytearray}(false) - scope-17 -> scope-27 | | -| Project[bytearray][0] - scope-14 +| Project[bytearray][0] - scope-18 | -|---a: Load(file:///tmp/pigoutput/Dir3:org.apache.pig.builtin.PigStorage) - scope-8 +|---c: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-10 Tez vertex scope-25 # Plan on vertex d: Local Rearrange[tuple]{bytearray}(false) - scope-15 -> scope-27 | | | Project[bytearray][0] - scope-16 | -|---b: Load(file:///tmp/pigoutput/Dir2:BinStorage) - scope-9 -Tez vertex scope-26 +|---b: Load(file:///tmp/output/Dir2:BinStorage) - scope-9 +Tez vertex scope-24 # Plan on vertex -d: Local Rearrange[tuple]{bytearray}(false) - scope-17 -> scope-27 +d: Local Rearrange[tuple]{bytearray}(false) - scope-13 -> scope-27 | | -| Project[bytearray][0] - scope-18 +| Project[bytearray][0] - scope-14 | -|---c: Load(file:///tmp/pigoutput/Dir1:org.apache.pig.builtin.PigStorage) - scope-10 +|---a: Load(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-8 Tez vertex scope-27 # Plan on vertex -d: Store(file:///tmp/pigoutput/Dir5:org.apache.pig.builtin.PigStorage) - scope-19 +d: Store(file:///tmp/output/Dir5:org.apache.pig.builtin.PigStorage) - scope-19 | |---d: Package(Packager)[tuple]{bytearray} - scope-12 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld Fri Feb 24 03:34:37 2017 @@ -26,7 +26,7 @@ POValueOutputTez - scope-58 -> [scope-5 |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-33 Tez vertex scope-59 # Plan on vertex -b: Store(file:///tmp/pigoutput/b:org.apache.pig.builtin.PigStorage) - scope-46 +b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-46 | |---b: Filter[bag] - scope-42 | | @@ -39,7 +39,7 @@ b: Store(file:///tmp/pigoutput/b:org.apa |---POValueInputTez - scope-60 <- scope-57 Tez vertex scope-61 # Plan on vertex -c: Store(file:///tmp/pigoutput/c:org.apache.pig.builtin.PigStorage) - scope-51 +c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-51 | |---c: Filter[bag] - scope-47 | | @@ -52,7 +52,7 @@ c: Store(file:///tmp/pigoutput/c:org.apa |---POValueInputTez - scope-62 <- scope-57 Tez vertex scope-63 # Plan on vertex -d: Store(file:///tmp/pigoutput/d:org.apache.pig.builtin.PigStorage) - scope-56 +d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-56 | |---d: Filter[bag] - scope-52 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld Fri Feb 24 03:34:37 2017 @@ -10,7 +10,7 @@ Tez vertex scope-24 # Plan on vertex 1-1: Split - scope-32 | | -| b: Store(file:///tmp/pigoutput/b:org.apache.pig.builtin.PigStorage) - scope-13 +| b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-13 | | | |---b: Filter[bag] - scope-9 | | | @@ -20,7 +20,7 @@ Tez vertex scope-24 | | | | | |---Constant(5) - scope-11 | | -| c: Store(file:///tmp/pigoutput/c:org.apache.pig.builtin.PigStorage) - scope-18 +| c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-18 | | | |---c: Filter[bag] - scope-14 | | | @@ -30,7 +30,7 @@ Tez vertex scope-24 | | | | | |---Constant(10) - scope-16 | | -| d: Store(file:///tmp/pigoutput/d:org.apache.pig.builtin.PigStorage) - scope-23 +| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-23 | | | |---d: Filter[bag] - scope-19 | | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Fri Feb 24 03:34:37 2017 @@ -72,12 +72,12 @@ POValueOutputTez - scope-325 -> [scope- |---b1: Package(Packager)[tuple]{int} - scope-228 Tez vertex scope-326 # Plan on vertex -b1: Store(file:///tmp/pigoutput/b1:org.apache.pig.builtin.PigStorage) - scope-234 +b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-234 | |---POValueInputTez - scope-327 <- scope-324 Tez vertex scope-328 # Plan on vertex -b2: Store(file:///tmp/pigoutput/b2:org.apache.pig.builtin.PigStorage) - scope-244 +b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-244 | |---b2: New For Each(false,false)[bag] - scope-243 | | @@ -119,7 +119,7 @@ c1: Local Rearrange[tuple]{int}(false) - |---POValueInputTez - scope-336 <- scope-319 Tez vertex scope-337 # Plan on vertex -c1: Store(file:///tmp/pigoutput/c1:org.apache.pig.builtin.PigStorage) - scope-265 +c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-265 | |---c1: New For Each(true,true)[tuple] - scope-264 | | @@ -163,7 +163,7 @@ c2: Local Rearrange[tuple]{int}(false) - | |---c2: Package(CombinerPackager)[tuple]{int} - scope-403 # Plan on vertex -c3: Store(file:///tmp/pigoutput/c1:org.apache.pig.builtin.PigStorage) - scope-279 +c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-279 | |---c3: New For Each(false,false)[bag] - scope-278 | | @@ -197,7 +197,7 @@ POValueOutputTez - scope-343 -> [scope- |---POValueInputTez - scope-342 <- scope-317 Tez vertex scope-344 # Plan on vertex -d1: Store(file:///tmp/pigoutput/d1:org.apache.pig.builtin.PigStorage) - scope-291 +d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-291 | |---POValueInputTez - scope-345 <- scope-341 Tez vertex scope-346 @@ -258,7 +258,7 @@ POIdentityInOutTez - scope-370 <- scope | Project[int][0] - scope-299 Tez vertex scope-371 # Plan on vertex -e1: Store(file:///tmp/pigoutput/e1:org.apache.pig.builtin.PigStorage) - scope-301 +e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-301 | |---New For Each(true)[tuple] - scope-374 | | @@ -289,17 +289,17 @@ POValueOutputTez - scope-381 -> [scope- |---POValueInputTez - scope-379 <- scope-375 Tez vertex scope-382 # Plan on vertex -f1: Store(file:///tmp/pigoutput/f1:org.apache.pig.builtin.PigStorage) - scope-310 +f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-310 | |---POValueInputTez - scope-383 <- scope-378 Tez vertex scope-384 # Plan on vertex -f2: Store(file:///tmp/pigoutput/f2:org.apache.pig.builtin.PigStorage) - scope-413 -> scope-316 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 -> scope-316 | |---POValueInputTez - scope-385 <- scope-341 Tez vertex scope-386 # Plan on vertex -f2: Store(file:///tmp/pigoutput/f2:org.apache.pig.builtin.PigStorage) - scope-414 -> scope-316 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 -> scope-316 | |---POValueInputTez - scope-387 <- scope-378 Tez vertex group scope-412 <- [scope-384, scope-386] -> null Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Fri Feb 24 03:34:37 2017 @@ -108,9 +108,9 @@ Tez vertex scope-106 | | | d1: Split - scope-206 | | | -| | d1: Store(file:///tmp/pigoutput/d1:org.apache.pig.builtin.PigStorage) - scope-80 +| | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80 | | | -| | f2: Store(file:///tmp/pigoutput/f2:org.apache.pig.builtin.PigStorage) - scope-209 -> scope-105 +| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 -> scope-105 | | | |---d1: Filter[bag] - scope-73 | | | @@ -143,9 +143,9 @@ Tez vertex scope-113 # Plan on vertex b1: Split - scope-201 | | -| b1: Store(file:///tmp/pigoutput/b1:org.apache.pig.builtin.PigStorage) - scope-23 +| b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23 | | -| b2: Store(file:///tmp/pigoutput/b2:org.apache.pig.builtin.PigStorage) - scope-33 +| b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-33 | | | |---b2: New For Each(false,false)[bag] - scope-32 | | | @@ -160,7 +160,7 @@ b1: Split - scope-201 |---b1: Package(Packager)[tuple]{int} - scope-17 Tez vertex scope-126 # Plan on vertex -c1: Store(file:///tmp/pigoutput/c1:org.apache.pig.builtin.PigStorage) - scope-54 +c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-54 | |---c1: New For Each(true,true)[tuple] - scope-53 | | @@ -185,7 +185,7 @@ c2: Local Rearrange[tuple]{int}(false) - | |---c2: Package(CombinerPackager)[tuple]{int} - scope-192 # Plan on vertex -c3: Store(file:///tmp/pigoutput/c1:org.apache.pig.builtin.PigStorage) - scope-68 +c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68 | |---c3: New For Each(false,false)[bag] - scope-67 | | @@ -220,7 +220,7 @@ POIdentityInOutTez - scope-159 <- scope | Project[int][0] - scope-88 Tez vertex scope-160 # Plan on vertex -e1: Store(file:///tmp/pigoutput/e1:org.apache.pig.builtin.PigStorage) - scope-90 +e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-90 | |---New For Each(true)[tuple] - scope-163 | | @@ -231,9 +231,9 @@ Tez vertex scope-167 # Plan on vertex f1: Split - scope-202 | | -| f1: Store(file:///tmp/pigoutput/f1:org.apache.pig.builtin.PigStorage) - scope-99 +| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99 | | -| f2: Store(file:///tmp/pigoutput/f2:org.apache.pig.builtin.PigStorage) - scope-210 -> scope-105 +| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 -> scope-105 | |---f1: Limit - scope-169 |
