Modified: pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java Thu Nov 27 12:49:54 2014 @@ -133,7 +133,7 @@ public class TestRegisteredJarVisibility @Test public void testRegisterJarVisibilityMR() throws IOException { PigServer pigServer = new PigServer(cluster.getExecType(), cluster.getProperties()); - testRegisteredJarVisibility(pigServer, INPUT_FILE.getName()); + testRegisteredJarVisibility(pigServer, INPUT_FILE.getAbsolutePath()); } @Test @@ -144,7 +144,7 @@ public class TestRegisteredJarVisibility public void testRegisteredJarVisibility(PigServer pigServer, String inputPath) throws IOException { String query = "register " + jarFile.getAbsolutePath() + ";\n" - + "a = load '" + inputPath + + "a = load '" + Util.generateURI(inputPath, pigServer.getPigContext()) + "' using org.apache.pig.test.RegisteredJarVisibilityLoader();\n" // register again to test classloader consistency + "register " + jarFile.getAbsolutePath() + ";\n"
Modified: pig/branches/spark/test/org/apache/pig/test/TestSchema.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestSchema.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestSchema.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestSchema.java Thu Nov 27 12:49:54 2014 @@ -20,6 +20,7 @@ package org.apache.pig.test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -123,6 +124,41 @@ public class TestSchema { } @Test + public void testParsingMapSchemasFromString() throws ParserException { + assertNotNull(Utils.getSchemaFromString("b:[(a:int)]")); + assertNotNull(Utils.getSchemaFromString("b:[someAlias: (b:int)]")); + assertNotNull(Utils.getSchemaFromString("a:map[{bag: (a:int)}]")); + assertNotNull(Utils.getSchemaFromString("a:map[someAlias: {bag: (a:int)}]")); + assertNotNull(Utils.getSchemaFromString("a:map[chararray]")); + assertNotNull(Utils.getSchemaFromString("a:map[someAlias: chararray]")); + assertNotNull(Utils.getSchemaFromString("a:map[someAlias: (bar: {bag: (a:int)})]")); + } + + @Test + public void testMapWithoutAlias() throws FrontendException { + List<FieldSchema> innerFields = new ArrayList<FieldSchema>(); + innerFields.add(new FieldSchema(null, DataType.LONG)); + List<FieldSchema> fields = new ArrayList<FieldSchema>(); + fields.add(new FieldSchema("mapAlias", new Schema(innerFields), DataType.MAP)); + + Schema inputSchema = new Schema(fields); + Schema fromString = Utils.getSchemaFromBagSchemaString(inputSchema.toString()); + assertTrue(Schema.equals(inputSchema, fromString, false, false)); + } + + @Test + public void testMapWithAlias() throws FrontendException { + List<FieldSchema> innerFields = new ArrayList<FieldSchema>(); + innerFields.add(new FieldSchema("valueAlias", DataType.LONG)); + List<FieldSchema> fields = new ArrayList<FieldSchema>(); + fields.add(new FieldSchema("mapAlias", new Schema(innerFields), DataType.MAP)); + + Schema inputSchema = new Schema(fields); + Schema fromString = Utils.getSchemaFromBagSchemaString(inputSchema.toString()); + assertTrue(Schema.equals(inputSchema, fromString, false, false)); + } + + @Test public void testNormalNestedMerge1() { // Generate two schemas @@ -879,10 +915,27 @@ public class TestSchema { } } - @Test(expected=ParserException.class) - public void testGetStringFromSchemaNegative() throws Exception { - String schemaString = "a:int b:long"; // A comma is missing between fields - Utils.getSchemaFromString(schemaString); - fail("The schema string is invalid, so parsing should fail!"); - } + @Test(expected = ParserException.class) + public void testGetStringFromSchemaNegative() throws Exception { + String schemaString = "a:int b:long"; // A comma is missing between fields + Utils.getSchemaFromString(schemaString); + fail("The schema string is invalid, so parsing should fail!"); + } + + @Test + public void testGetInitialSchemaStringFromSchema() throws ParserException { + String[] schemaStrings = { + "my_list:{array:(array_element:(num1:int,num2:int))}", + "my_list:{array:(array_element:(num1:int,num2:int),c:chararray)}", + "bag:{mytuple3:(mytuple2:(mytuple:(f1:int)))}", + "bag:{mytuple:(f1:int)}", + "{((num1:int,num2:int))}" + }; + for (String schemaString : schemaStrings) { + String s1 = Utils.getSchemaFromString(schemaString).toString(); + //check if we get back the initial schema string + String s2 = s1.substring(1, s1.length() - 1).replaceAll("\\s|bag_0:|tuple_0:", ""); + assertTrue(schemaString.equals(s2)); + } + } } Modified: pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java Thu Nov 27 12:49:54 2014 @@ -662,7 +662,8 @@ public class TestScriptLanguage { PigStatsUtil.getEmptyPigStats(); // ExecMode.FILE - stats = PigRunner.run(new String[] { "-f", scriptFile.getAbsolutePath(), "arg0", + stats = PigRunner.run(new String[] { "-x", cluster.getExecType().name().toLowerCase(), + "-f", scriptFile.getAbsolutePath(), "arg0", file1 + "," + file2 }, null); assertEquals(null, stats.getErrorMessage()); assertFileNotExists(file1, file2); Modified: pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java Thu Nov 27 12:49:54 2014 @@ -231,7 +231,7 @@ public class TestScriptUDF{ String userenv = Shell.WINDOWS?"USERNAME":"USER"; String[] input = { userenv, - "PATH" + "JAVA_HOME" }; Util.createInputFile(cluster, "testPythonBuiltinModuleImport1", input); Modified: pig/branches/spark/test/org/apache/pig/test/TestStore.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStore.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStore.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStore.java Thu Nov 27 12:49:54 2014 @@ -82,7 +82,6 @@ import org.apache.pig.parser.ParserExcep import org.apache.pig.parser.QueryParserDriver; import org.apache.pig.test.utils.GenRandomData; import org.apache.pig.test.utils.TestHelper; -import org.joda.time.DateTimeZone; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -116,7 +115,6 @@ public class TestStore { inputFileName = TESTDIR + "/TestStore-" + new Random().nextLong() + ".txt"; outputFileName = TESTDIR + "/TestStore-output-" + new Random().nextLong() + ".txt"; - DateTimeZone.setDefault(DateTimeZone.forOffsetMillis(DateTimeZone.UTC.getOffset(null))); } @After @@ -355,7 +353,7 @@ public class TestStore { t.append(flds[9].compareTo("")!=0 ? ps.getLoadCaster().bytesToBoolean(flds[9].getBytes()) : null); t.append(flds[10].compareTo("")!=0 ? ps.getLoadCaster().bytesToDateTime(flds[10].getBytes()) : null); t.append(flds[11].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[10].getBytes()) : null); - assertTrue(TestHelper.tupleEquals(inputTuple, t)); + assertEquals(inputTuple, t); } br.close(); } @@ -904,7 +902,7 @@ public class TestStore { } private void checkStorePath(String orig, String expected, boolean isTmp) throws Exception { - pc.getProperties().setProperty(PigConfiguration.OPT_MULTIQUERY,""+true); + pc.getProperties().setProperty(PigConfiguration.PIG_OPT_MULTIQUERY,""+true); DataStorage dfs = pc.getDfs(); dfs.setActiveContainer(dfs.asContainer("/tmp")); Modified: pig/branches/spark/test/org/apache/pig/test/TestStreaming.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreaming.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStreaming.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStreaming.java Thu Nov 27 12:49:54 2014 @@ -113,7 +113,7 @@ public class TestStreaming { // Pig query to run pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); @@ -157,7 +157,7 @@ public class TestStreaming { } // Pig query to run pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); @@ -200,7 +200,7 @@ public class TestStreaming { // Pig query to run pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); @@ -250,7 +250,7 @@ public class TestStreaming { // Pig query to run pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); @@ -318,7 +318,7 @@ public class TestStreaming { "output(stdout using " + PigStreaming.class.getName() + "(',')) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using PigStorage(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;"); @@ -388,7 +388,7 @@ public class TestStreaming { "output(stdout using PS ) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using PigStorage(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;"); pigServer.registerQuery("STREAMED_DATA = stream FILTERED_DATA " + @@ -458,7 +458,7 @@ public class TestStreaming { "input('bar' using " + PigStreaming.class.getName() + "(',')) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;"); @@ -520,7 +520,7 @@ public class TestStreaming { "'bar' using " + PigStreaming.class.getName() + "(',')) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;"); @@ -583,7 +583,7 @@ public class TestStreaming { "'bar' using PS) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using PigStorage(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;"); pigServer.registerQuery("OP = stream FILTERED_DATA through CMD;"); @@ -644,7 +644,7 @@ public class TestStreaming { "'foobar' using " + PigStreaming.class.getName() + "(',')) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;"); @@ -697,7 +697,7 @@ public class TestStreaming { pigServer.registerQuery("define CMD `" + simpleEchoStreamingCommand + " | " + simpleEchoStreamingCommand + "`;"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); if(withTypes[i] == true) { @@ -738,7 +738,7 @@ public class TestStreaming { pigServer.registerQuery("define CMD `"+ simpleEchoStreamingCommand + "` input(stdin using " + PigStreamDump.class.getName() + ");"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); @@ -857,7 +857,7 @@ public class TestStreaming { "output(stdout using " + PigStreaming.class.getName() + "(',')) " + "stderr();"); pigServer.registerQuery("IP = load '" - + Util.generateURI(Util.encodeEscape(input.toString()), + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using PigStorage();"); pigServer.registerQuery("STREAMED_DATA = stream IP through CMD;"); Modified: pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java Thu Nov 27 12:49:54 2014 @@ -98,7 +98,7 @@ public class TestStreamingLocal { // Pig query to run pigServer.registerQuery("IP = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); pigServer.registerQuery("S1 = stream FILTERED_DATA through `" + @@ -141,7 +141,7 @@ public class TestStreamingLocal { } // Pig query to run pigServer.registerQuery("IP = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); if(withTypes[i] == true) { @@ -183,7 +183,7 @@ public class TestStreamingLocal { // Pig query to run pigServer.registerQuery("IP = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); pigServer.registerQuery("GROUPED_DATA = group FILTERED_DATA by $0;"); @@ -232,7 +232,7 @@ public class TestStreamingLocal { // Pig query to run pigServer.registerQuery("IP = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); pigServer.registerQuery("S1 = stream FILTERED_DATA through `" + @@ -280,7 +280,7 @@ public class TestStreamingLocal { pigServer.registerQuery("define CMD `" + simpleEchoStreamingCommand + " | " + simpleEchoStreamingCommand + "`;"); pigServer.registerQuery("IP = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); if(withTypes[i] == true) { pigServer.registerQuery("OP = stream IP through CMD as (f0:chararray, f1:int);"); @@ -310,11 +310,11 @@ public class TestStreamingLocal { expected.set(3, 0); pigServer.registerQuery("A = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',') as (a0, a1);"); pigServer.registerQuery("B = stream A through `head -1` as (a0, a1);"); pigServer.registerQuery("C = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',') as (a0, a1);"); pigServer.registerQuery("D = stream C through `head -1` as (a0, a1);"); pigServer.registerQuery("E = join B by a0, D by a0;"); @@ -359,7 +359,7 @@ public class TestStreamingLocal { pigServer.registerQuery("define CMD `"+ simpleEchoStreamingCommand + "` input(stdin);"); pigServer.registerQuery("IP = load '" + - Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) + + Util.generateURI(input.toString(), pigServer.getPigContext()) + "' using " + PigStorage.class.getName() + "(',');"); pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';"); if(withTypes[i] == true) { Modified: pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java Thu Nov 27 12:49:54 2014 @@ -32,11 +32,13 @@ import org.apache.pig.builtin.REPLACE; import org.apache.pig.builtin.STARTSWITH; import org.apache.pig.builtin.ENDSWITH; import org.apache.pig.builtin.STRSPLIT; +import org.apache.pig.builtin.STRSPLITTOBAG; import org.apache.pig.builtin.SUBSTRING; import org.apache.pig.builtin.TRIM; import org.apache.pig.builtin.LTRIM; import org.apache.pig.builtin.RTRIM; import org.apache.pig.builtin.EqualsIgnoreCase; +import org.apache.pig.data.DataBag; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.junit.Test; @@ -227,6 +229,77 @@ public class TestStringUDFs { } @Test + public void testSplitToBag() throws IOException { + STRSPLITTOBAG bagSplit = new STRSPLITTOBAG(); + + //test no delims in input + Tuple testTuple = Util.buildTuple("1 2 3", "4"); + DataBag outputBag = bagSplit.exec(testTuple); + assertEquals("No of records split should be 1", 1, outputBag.size()); + assertEquals("Split string should match the input string", "(1 2 3)", outputBag.iterator().next().toString()); + + //test default delimiter + testTuple = Util.buildTuple("1 2 3"); + outputBag = bagSplit.exec(testTuple); + String[] assertionArray = {"1", "2", "3"}; + assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size()); + + int i = 0; + for (Tuple t : outputBag) { + assertEquals("Assertion tests on split strings", "(" + assertionArray[i] + ")", t.toString()); + i++; + } + + //test split on specified delimiter + testTuple = Util.buildTuple("1:2:3", ":"); + outputBag = bagSplit.exec(testTuple); + assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size()); + i = 0; + for (Tuple t : outputBag) { + assertEquals("Assertion tests on split strings", "(" + assertionArray[i] + ")", t.toString()); + i++; + } + + // test limiting results with limit + testTuple = Util.buildTuple("1:2:3", ":", 2); + outputBag = bagSplit.exec(testTuple); + assertionArray = new String[]{"1", "2:3"}; + assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size()); + i = 0; + for (Tuple t : outputBag) { + assertEquals("Matched records in split results with limit", "(" + assertionArray[i] + ")", t.toString()); + i++; + } + + // test trimming of whitespace + testTuple = Util.buildTuple("1 2 "); + outputBag = bagSplit.exec(testTuple); + assertionArray = new String[]{"1", "2"}; + assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size()); + i = 0; + for (Tuple t : outputBag) { + assertEquals("Matched records in split results with trimming of whitespaces", "(" + assertionArray[i] + ")", t.toString()); + i++; + } + + // test forcing null matches with length param + testTuple = Util.buildTuple("1:2:::", ":", 10); + outputBag = bagSplit.exec(testTuple); + assertionArray = new String[]{"1", "2", "", "", ""}; + assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size()); + i = 0; + for (Tuple t : outputBag) { + assertEquals("Matched records in split results with forcing null matched with limit", "(" + assertionArray[i] + ")", t.toString()); + i++; + } + + //test wrong schemas + testTuple = Util.buildTuple(1, 2, 3); + outputBag = bagSplit.exec(testTuple); + assertEquals("Wrong Schema checks", null, outputBag); + } + + @Test public void testStartsWith() throws IOException { STARTSWITH startsWith = new STARTSWITH(); Tuple testTuple1 = Util.buildTuple("foo", "bar"); Modified: pig/branches/spark/test/org/apache/pig/test/TestTuple.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTuple.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestTuple.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestTuple.java Thu Nov 27 12:49:54 2014 @@ -81,31 +81,31 @@ public class TestTuple { public void testEmptyTupleSize() { Tuple t = mTupleFactory.newTuple(); long size = t.getMemorySize(); - assertEquals("tuple size",size, 96); + assertEquals("tuple size", 88, size); } - + @Test public void testEmptyBagSize() { DataBag bag = DefaultBagFactory.getInstance().newDefaultBag(); long size = bag.getMemorySize(); - assertEquals("bag size",size, 124); + assertEquals("bag size", 124, size); } - + @Test // See PIG-1443 public void testTupleSizeWithString() { Tuple t = Util.createTuple(new String[] {"1234567", "bar"}); long size = t.getMemorySize(); - assertEquals("tuple size",size, 200); + assertEquals("tuple size", 192, size); } - + @Test public void testTupleSizeWithByteArrays() { Tuple t = mTupleFactory.newTuple(); t.append(new DataByteArray("1234567")); t.append(new DataByteArray("bar")); long size = t.getMemorySize(); - assertEquals("tuple size",size, 168); + assertEquals("tuple size", 160, size); } @Test @@ -114,7 +114,7 @@ public class TestTuple { t.append(new Double(0.1)); t.append(new Double(2000.10001)); long size = t.getMemorySize(); - assertEquals("tuple size",size, 128); + assertEquals("tuple size", 120, size); } @Test @@ -123,27 +123,27 @@ public class TestTuple { t.append(new Float(0.1F)); t.append(new Float(2000.10001F)); long size = t.getMemorySize(); - assertEquals("tuple size",size, 128); + assertEquals("tuple size", 120, size); } - + @Test public void testTupleSizeWithLongs() { Tuple t = mTupleFactory.newTuple(); t.append(new Long(100)); t.append(new Long(2000)); long size = t.getMemorySize(); - assertEquals("tuple size",size, 128); + assertEquals("tuple size", 120, size); } - + @Test public void testTupleSizeWithBooleans() { Tuple t = mTupleFactory.newTuple(); t.append(new Boolean(true)); t.append(new Boolean(false)); long size = t.getMemorySize(); - assertEquals("tuple size",size, 128); - } - + assertEquals("tuple size", 120, size); + } + @Test public void testTupleIterator() { Tuple t = mTupleFactory.newTuple(); Modified: pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Thu Nov 27 12:49:54 2014 @@ -3745,7 +3745,7 @@ public class TestTypeCheckingValidatorNe "[k1#good,k2#morning]" }; PigServer ps = new PigServer(ExecType.LOCAL); File f = org.apache.pig.test.Util.createInputFile("test", ".txt", input); - String inputFileName = Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), ps.getPigContext()); + String inputFileName = Util.generateURI(f.getAbsolutePath(), ps.getPigContext()); // load as bytearray and use as map String query = "a= load '" + inputFileName + "' as (m);" + " b = foreach a generate m#'k1';"; Modified: pig/branches/spark/test/org/apache/pig/test/TestUDF.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUDF.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestUDF.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestUDF.java Thu Nov 27 12:49:54 2014 @@ -20,6 +20,7 @@ package org.apache.pig.test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import java.io.File; import java.io.FileWriter; @@ -319,4 +320,31 @@ public class TestUDF { return l; } } -} \ No newline at end of file + + @Test + // See PIG-4184 + public void testUDFNullInput() throws Exception { + PigServer pig = new PigServer(ExecType.LOCAL); + File inputFile = Util.createInputFile("tmp", "", + new String[] {"\t", "2\t3"}); + pig.registerQuery("a = load '" + + Util.generateURI(inputFile.toString(), pig.getPigContext()) + + "' as (i1:int, i2:int);"); + pig.registerQuery("b = foreach a generate " + IntToBool.class.getName() + "(i1);"); + + Iterator<Tuple> iter = pig.openIterator("b"); + assertEquals(iter.next().toString(), "(false)"); + assertEquals(iter.next().toString(), "(true)"); + assertFalse(iter.hasNext()); + } + + public static class IntToBool extends EvalFunc<Boolean> { + @Override + public Boolean exec(Tuple input) throws IOException { + if (input == null || input.size() == 0) + return null; + Integer val = (Integer)input.get(0); + return (val == null || val == 0) ? false : true; + } + } +} Modified: pig/branches/spark/test/org/apache/pig/test/TestUnion.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUnion.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestUnion.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestUnion.java Thu Nov 27 12:49:54 2014 @@ -95,7 +95,7 @@ public class TestUnion { POLoad ld1 = GenPhyOp.topLoadOp(); String curDir = System.getProperty("user.dir"); String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/"; - FileSpec fSpec = new FileSpec(Util.generateURI(Util.encodeEscape(inpDir + "passwd"), pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"})); + FileSpec fSpec = new FileSpec(Util.generateURI(inpDir + "passwd", pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"})); ld1.setLFile(fSpec); POLoad ld2 = GenPhyOp.topLoadOp(); @@ -290,8 +290,8 @@ public class TestUnion { PigServer ps = new PigServer(ExecType.LOCAL, new Properties()); //PigStorage and TextLoader have different LoadCasters - ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);"); - ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (b:bytearray);"); + ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a:bytearray);"); + ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' using TextLoader() as (b:bytearray);"); ps.registerQuery("C = union onschema A,B;"); ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;"); @@ -321,9 +321,9 @@ public class TestUnion { File f3 = Util.createInputFile("tmp", "i3.txt", new String[] {"1","2","3"}); PigServer ps = new PigServer(ExecType.LOCAL, new Properties()); - ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);"); // Using PigStorage() - ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (i:bytearray);"); - ps.registerQuery("C = load '" + f3.getAbsolutePath() + "' using TextLoader() as (i:bytearray);"); + ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a:bytearray);"); // Using PigStorage() + ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' using TextLoader() as (i:bytearray);"); + ps.registerQuery("C = load '" + Util.encodeEscape(f3.getAbsolutePath()) + "' using TextLoader() as (i:bytearray);"); ps.registerQuery("B2 = join B by i, A by a;"); //{A::a: bytearray,B::i: bytearray} ps.registerQuery("B3 = foreach B2 generate a, B::i as i;"); //{A::a: bytearray,i: bytearray} ps.registerQuery("C2 = join C by i, A by a;"); //{A::a: bytearray,C::i: bytearray} @@ -347,8 +347,8 @@ public class TestUnion { PigServer ps = new PigServer(ExecType.LOCAL, new Properties()); // PigStorage and PigStorageWithStatistics have the same // LoadCaster(== Utf8StorageConverter) - ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray, b:bytearray);"); - ps.registerQuery("B = load '" + f1.getAbsolutePath() + + ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a:bytearray, b:bytearray);"); + ps.registerQuery("B = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' using org.apache.pig.test.PigStorageWithStatistics() as (a:bytearray, b:bytearray);"); ps.registerQuery("C = union onschema A,B;"); ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;"); Modified: pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java Thu Nov 27 12:49:54 2014 @@ -492,7 +492,7 @@ public class TestUnionOnSchema { "(4,5,6,null)", }); - Util.compareActualAndExpectedResults(list1, expectedRes); + Util.checkQueryOutputsAfterSort(list1, expectedRes); assertEquals(0, list2.size()); } @@ -852,6 +852,7 @@ public class TestUnionOnSchema { * Udf that has schema of tuple column with no inner schema */ public static class UDFTupleNullSchema extends EvalFunc <Tuple> { + @Override public Tuple exec(Tuple input) { return input; } Modified: pig/branches/spark/test/org/apache/pig/test/Util.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/Util.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/Util.java (original) +++ pig/branches/spark/test/org/apache/pig/test/Util.java Thu Nov 27 12:49:54 2014 @@ -37,6 +37,7 @@ import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringReader; +import java.io.Writer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -53,10 +54,13 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.log4j.Appender; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; import org.apache.log4j.SimpleLayout; +import org.apache.log4j.WriterAppender; import org.apache.pig.ExecType; import org.apache.pig.ExecTypeProvider; import org.apache.pig.LoadCaster; @@ -540,14 +544,10 @@ public class Util { while(actualResultsIt.hasNext()){ actualResList.add(actualResultsIt.next()); } - - compareActualAndExpectedResults(actualResList, expectedResList); - + checkQueryOutputsAfterSort(actualResList, expectedResList); } - - - static public void compareActualAndExpectedResults( + static public void checkQueryOutputsAfterSort( List<Tuple> actualResList, List<Tuple> expectedResList) { Collections.sort(actualResList); Collections.sort(expectedResList); @@ -611,7 +611,7 @@ public class Util { } static private String getMkDirCommandForHadoop2_0(String fileName) { - if (Util.isHadoop23() || Util.isHadoop2_0()) { + if (org.apache.pig.impl.util.Utils.isHadoop23() || org.apache.pig.impl.util.Utils.isHadoop2()) { Path parentDir = new Path(fileName).getParent(); String mkdirCommand = parentDir.getName().isEmpty() ? "" : "fs -mkdir -p " + parentDir + "\n"; return mkdirCommand; @@ -630,7 +630,11 @@ public class Util { static public void copyFromLocalToCluster(MiniGenericCluster cluster, String localFileName, String fileNameOnCluster) throws IOException { if(Util.WINDOWS){ - localFileName = localFileName.replace('\\','/'); + if (!localFileName.contains(":")) { + localFileName = localFileName.replace('\\','/'); + } else { + localFileName = localFileName.replace('/','\\'); + } fileNameOnCluster = fileNameOnCluster.replace('\\','/'); } PigServer ps = new PigServer(cluster.getExecType(), cluster.getProperties()); @@ -912,6 +916,38 @@ public class Util { return executeJavaCommandAndReturnInfo(cmd).exitCode; } + public static class ReadStream implements Runnable { + InputStream is; + Thread thread; + String message = ""; + public ReadStream(InputStream is) { + this.is = is; + } + public void start () { + thread = new Thread (this); + thread.start (); + } + public void run () { + try { + InputStreamReader isr = new InputStreamReader (is); + BufferedReader br = new BufferedReader (isr); + while (true) { + String s = br.readLine (); + if (s == null) break; + if (!message.isEmpty()) { + message += "\n"; + } + message += s; + } + is.close (); + } catch (Exception ex) { + ex.printStackTrace (); + } + } + public String getMessage() { + return message; + } + } public static ProcessReturnInfo executeJavaCommandAndReturnInfo(String cmd) throws Exception { @@ -922,24 +958,17 @@ public class Util { } Process cmdProc = Runtime.getRuntime().exec(cmd); ProcessReturnInfo pri = new ProcessReturnInfo(); - pri.stderrContents = getContents(cmdProc.getErrorStream()); - pri.stdoutContents = getContents(cmdProc.getInputStream()); + ReadStream stdoutStream = new ReadStream(cmdProc.getInputStream ()); + ReadStream stderrStream = new ReadStream(cmdProc.getErrorStream ()); + stdoutStream.start(); + stderrStream.start(); cmdProc.waitFor(); pri.exitCode = cmdProc.exitValue(); + pri.stdoutContents = stdoutStream.getMessage(); + pri.stderrContents = stderrStream.getMessage(); return pri; } - private static String getContents(InputStream istr) throws IOException { - BufferedReader br = new BufferedReader( - new InputStreamReader(istr)); - String s = ""; - String line; - while ( (line = br.readLine()) != null) { - s += line + "\n"; - } - return s; - - } public static class ProcessReturnInfo { public int exitCode; public String stderrContents; @@ -1232,13 +1261,6 @@ public class Util { return plan.replaceAll("','','[^']*','scope','true'\\)\\)", "','','','scope','true'))"); } - public static boolean isHadoop23() { - String version = org.apache.hadoop.util.VersionInfo.getVersion(); - if (version.matches("\\b0\\.23\\..+\\b")) - return true; - return false; - } - public static boolean isHadoop203plus() { String version = org.apache.hadoop.util.VersionInfo.getVersion(); if (version.matches("\\b0\\.20\\.2\\b")) @@ -1275,13 +1297,6 @@ public class Util { assertEquals("Unexpected value found in configs for " + param, expected, conf.getLong(param, -1)); } - public static boolean isHadoop2_0() { - String version = org.apache.hadoop.util.VersionInfo.getVersion(); - if (version.matches("\\b2\\.\\d\\..+")) - return true; - return false; - } - /** * Returns a PathFilter that filters out filenames that start with _. * @return PathFilter @@ -1357,4 +1372,18 @@ public class Util { return ExecTypeProvider.fromString("local"); } } + + public static void createLogAppender(Class clazz, String appenderName, Writer writer) { + Logger logger = Logger.getLogger(clazz); + WriterAppender writerAppender = new WriterAppender(new PatternLayout("%d [%t] %-5p %c %x - %m%n"), writer); + writerAppender.setName(appenderName); + logger.addAppender(writerAppender); + } + + public static void removeLogAppender(Class clazz, String appenderName) { + Logger logger = Logger.getLogger(clazz); + Appender appender = logger.getAppender(appenderName); + appender.close(); + logger.removeAppender(appenderName); + } } Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld Thu Nov 27 12:49:54 2014 @@ -2,64 +2,64 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: scope-66 +# TEZ DAG plan: pig-0_scope-1 #-------------------------------------------------- -Tez vertex scope-58 -> Tez vertex scope-60,Tez vertex scope-62,Tez vertex scope-64, -Tez vertex scope-64 -Tez vertex scope-62 -Tez vertex scope-60 +Tez vertex scope-57 -> Tez vertex scope-59,Tez vertex scope-61,Tez vertex scope-63, +Tez vertex scope-59 +Tez vertex scope-63 +Tez vertex scope-61 -Tez vertex scope-58 +Tez vertex scope-57 # Plan on vertex -POValueOutputTez - scope-59 -> [scope-64, scope-62, scope-60] +POValueOutputTez - scope-58 -> [scope-59, scope-63, scope-61] | -|---a: New For Each(false,false)[bag] - scope-41 +|---a: New For Each(false,false)[bag] - scope-40 | | - | Cast[int] - scope-36 + | Cast[int] - scope-35 | | - | |---Project[bytearray][0] - scope-35 + | |---Project[bytearray][0] - scope-34 | | - | Cast[int] - scope-39 + | Cast[int] - scope-38 | | - | |---Project[bytearray][1] - scope-38 + | |---Project[bytearray][1] - scope-37 | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-34 -Tez vertex scope-64 + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-33 +Tez vertex scope-59 # Plan on vertex -d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-57 +b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-46 | -|---d: Filter[bag] - scope-53 +|---b: Filter[bag] - scope-42 | | - | Greater Than[boolean] - scope-56 + | Less Than or Equal[boolean] - scope-45 | | - | |---Project[int][0] - scope-54 + | |---Project[int][0] - scope-43 | | - | |---Constant(10) - scope-55 + | |---Constant(5) - scope-44 | - |---POValueInputTez - scope-65 <- scope-58 -Tez vertex scope-62 + |---POValueInputTez - scope-60 <- scope-57 +Tez vertex scope-63 # Plan on vertex -c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-52 +d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-56 | -|---c: Filter[bag] - scope-48 +|---d: Filter[bag] - scope-52 | | - | Less Than or Equal[boolean] - scope-51 + | Greater Than[boolean] - scope-55 | | - | |---Project[int][0] - scope-49 + | |---Project[int][0] - scope-53 | | - | |---Constant(10) - scope-50 + | |---Constant(10) - scope-54 | - |---POValueInputTez - scope-63 <- scope-58 -Tez vertex scope-60 + |---POValueInputTez - scope-64 <- scope-57 +Tez vertex scope-61 # Plan on vertex -b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-47 +c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-51 | -|---b: Filter[bag] - scope-43 +|---c: Filter[bag] - scope-47 | | - | Less Than or Equal[boolean] - scope-46 + | Less Than or Equal[boolean] - scope-50 | | - | |---Project[int][0] - scope-44 + | |---Project[int][0] - scope-48 | | - | |---Constant(5) - scope-45 + | |---Constant(10) - scope-49 | - |---POValueInputTez - scope-61 <- scope-58 + |---POValueInputTez - scope-62 <- scope-57 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld Thu Nov 27 12:49:54 2014 @@ -2,13 +2,13 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: scope-32 +# TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- Tez vertex scope-24 Tez vertex scope-24 # Plan on vertex -1-1: Split - scope-33 +1-1: Split - scope-32 | | | b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-13 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Thu Nov 27 12:49:54 2014 @@ -2,305 +2,305 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: scope-393 +# TEZ DAG plan: pig-0_scope-1 #-------------------------------------------------- -Tez vertex scope-318 -> Tez vertex scope-320,Tez vertex scope-331,Tez vertex scope-342, -Tez vertex scope-320 -> Tez vertex scope-323,Tez vertex scope-336,Tez vertex scope-347, -Tez vertex scope-347 -> Tez vertex scope-350,Tez vertex scope-376, -Tez vertex scope-376 -> Tez vertex scope-379, -Tez vertex scope-379 -> Tez vertex scope-383,Tez vertex scope-387, -Tez vertex scope-383 -Tez vertex scope-342 -> Tez vertex scope-345,Tez vertex scope-385, -Tez vertex scope-385 -> Tez vertex group scope-414, -Tez vertex scope-387 -> Tez vertex group scope-414, -Tez vertex group scope-414 -Tez vertex scope-331 -> Tez vertex scope-334,Tez vertex scope-339, -Tez vertex scope-339 -> Tez vertex scope-341, -Tez vertex scope-341 -Tez vertex scope-323 -> Tez vertex scope-325, -Tez vertex scope-325 -> Tez vertex scope-327,Tez vertex scope-329, -Tez vertex scope-327 -Tez vertex scope-345 -Tez vertex scope-329 -Tez vertex scope-350 -> Tez vertex scope-370,Tez vertex scope-360, -Tez vertex scope-360 -> Tez vertex scope-370, -Tez vertex scope-370 -> Tez vertex scope-372, -Tez vertex scope-372 -Tez vertex scope-334 -> Tez vertex scope-338, -Tez vertex scope-336 -> Tez vertex scope-338, -Tez vertex scope-338 +Tez vertex scope-317 -> Tez vertex scope-319,Tez vertex scope-330,Tez vertex scope-341, +Tez vertex scope-341 -> Tez vertex scope-344,Tez vertex scope-384, +Tez vertex scope-384 -> Tez vertex group scope-412, +Tez vertex scope-319 -> Tez vertex scope-322,Tez vertex scope-335,Tez vertex scope-346, +Tez vertex scope-346 -> Tez vertex scope-349,Tez vertex scope-375, +Tez vertex scope-375 -> Tez vertex scope-378, +Tez vertex scope-378 -> Tez vertex scope-382,Tez vertex scope-386, +Tez vertex scope-386 -> Tez vertex group scope-412, +Tez vertex group scope-412 +Tez vertex scope-382 +Tez vertex scope-330 -> Tez vertex scope-333,Tez vertex scope-338, +Tez vertex scope-338 -> Tez vertex scope-340, +Tez vertex scope-340 +Tez vertex scope-322 -> Tez vertex scope-324, +Tez vertex scope-324 -> Tez vertex scope-326,Tez vertex scope-328, +Tez vertex scope-326 +Tez vertex scope-344 +Tez vertex scope-328 +Tez vertex scope-349 -> Tez vertex scope-369,Tez vertex scope-359, +Tez vertex scope-359 -> Tez vertex scope-369, +Tez vertex scope-369 -> Tez vertex scope-371, +Tez vertex scope-371 +Tez vertex scope-333 -> Tez vertex scope-337, +Tez vertex scope-335 -> Tez vertex scope-337, +Tez vertex scope-337 -Tez vertex scope-318 +Tez vertex scope-317 # Plan on vertex -POValueOutputTez - scope-319 -> [scope-320, scope-331, scope-342] +POValueOutputTez - scope-318 -> [scope-341, scope-319, scope-330] | -|---a: New For Each(false,false)[bag] - scope-219 +|---a: New For Each(false,false)[bag] - scope-218 | | - | Cast[int] - scope-214 + | Cast[int] - scope-213 | | - | |---Project[bytearray][0] - scope-213 + | |---Project[bytearray][0] - scope-212 | | - | Cast[int] - scope-217 + | Cast[int] - scope-216 | | - | |---Project[bytearray][1] - scope-216 + | |---Project[bytearray][1] - scope-215 | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-212 -Tez vertex scope-320 + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-211 +Tez vertex scope-341 # Plan on vertex -POValueOutputTez - scope-322 -> [scope-336, scope-323, scope-347] +POValueOutputTez - scope-343 -> [scope-384, scope-344] | -|---b: Filter[bag] - scope-221 +|---d1: Filter[bag] - scope-284 | | - | Less Than or Equal[boolean] - scope-224 + | Equal To[boolean] - scope-287 | | - | |---Project[int][0] - scope-222 + | |---Project[int][0] - scope-285 | | - | |---Constant(5) - scope-223 + | |---Constant(5) - scope-286 | - |---POValueInputTez - scope-321 <- scope-318 -Tez vertex scope-347 -# Plan on vertex -POValueOutputTez - scope-349 -> [scope-376, scope-350] -| -|---POValueInputTez - scope-348 <- scope-320 -Tez vertex scope-376 -# Plan on vertex -POValueOutputTez - scope-378 -> [scope-379] -| -|---f1: Limit - scope-307 - | - |---f: Filter[bag] - scope-303 + |---d: Filter[bag] - scope-280 | | - | Greater Than or Equal[boolean] - scope-306 + | Greater Than[boolean] - scope-283 | | - | |---Project[int][0] - scope-304 + | |---Project[int][0] - scope-281 | | - | |---Constant(3) - scope-305 + | |---Constant(10) - scope-282 | - |---POValueInputTez - scope-377 <- scope-347 -Tez vertex scope-379 -# Plan on vertex -POValueOutputTez - scope-382 -> [scope-387, scope-383] -| -|---f1: Limit - scope-381 - | - |---POValueInputTez - scope-380 <- scope-376 -Tez vertex scope-383 + |---POValueInputTez - scope-342 <- scope-317 +Tez vertex scope-384 # Plan on vertex -f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-311 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 | -|---POValueInputTez - scope-384 <- scope-379 -Tez vertex scope-342 +|---POValueInputTez - scope-385 <- scope-341 +Tez vertex scope-319 # Plan on vertex -POValueOutputTez - scope-344 -> [scope-385, scope-345] +POValueOutputTez - scope-321 -> [scope-335, scope-322, scope-346] | -|---d1: Filter[bag] - scope-285 +|---b: Filter[bag] - scope-220 | | - | Equal To[boolean] - scope-288 + | Less Than or Equal[boolean] - scope-223 | | - | |---Project[int][0] - scope-286 + | |---Project[int][0] - scope-221 | | - | |---Constant(5) - scope-287 + | |---Constant(5) - scope-222 | - |---d: Filter[bag] - scope-281 + |---POValueInputTez - scope-320 <- scope-317 +Tez vertex scope-346 +# Plan on vertex +POValueOutputTez - scope-348 -> [scope-375, scope-349] +| +|---POValueInputTez - scope-347 <- scope-319 +Tez vertex scope-375 +# Plan on vertex +POValueOutputTez - scope-377 -> [scope-378] +| +|---f1: Limit - scope-306 + | + |---f: Filter[bag] - scope-302 | | - | Greater Than[boolean] - scope-284 + | Greater Than or Equal[boolean] - scope-305 | | - | |---Project[int][0] - scope-282 + | |---Project[int][0] - scope-303 | | - | |---Constant(10) - scope-283 + | |---Constant(3) - scope-304 | - |---POValueInputTez - scope-343 <- scope-318 -Tez vertex scope-385 + |---POValueInputTez - scope-376 <- scope-346 +Tez vertex scope-378 # Plan on vertex -f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-415 +POValueOutputTez - scope-381 -> [scope-386, scope-382] | -|---POValueInputTez - scope-386 <- scope-342 -Tez vertex scope-387 +|---f1: Limit - scope-380 + | + |---POValueInputTez - scope-379 <- scope-375 +Tez vertex scope-386 # Plan on vertex -f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-416 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 | -|---POValueInputTez - scope-388 <- scope-379 -Tez vertex group scope-414 <- [scope-385, scope-387] -> null +|---POValueInputTez - scope-387 <- scope-378 +Tez vertex group scope-412 <- [scope-384, scope-386] -> null # No plan on vertex group -Tez vertex scope-331 +Tez vertex scope-382 # Plan on vertex -POValueOutputTez - scope-333 -> [scope-339, scope-334] +f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-310 | -|---c: Filter[bag] - scope-246 +|---POValueInputTez - scope-383 <- scope-378 +Tez vertex scope-330 +# Plan on vertex +POValueOutputTez - scope-332 -> [scope-338, scope-333] +| +|---c: Filter[bag] - scope-245 | | - | Less Than or Equal[boolean] - scope-249 + | Less Than or Equal[boolean] - scope-248 | | - | |---Project[int][0] - scope-247 + | |---Project[int][0] - scope-246 | | - | |---Constant(10) - scope-248 + | |---Constant(10) - scope-247 | - |---POValueInputTez - scope-332 <- scope-318 -Tez vertex scope-339 + |---POValueInputTez - scope-331 <- scope-317 +Tez vertex scope-338 # Plan on vertex -c2: Local Rearrange[tuple]{int}(false) - scope-406 -> scope-341 +c2: Local Rearrange[tuple]{int}(false) - scope-404 -> scope-340 | | -| Project[int][0] - scope-408 +| Project[int][0] - scope-406 | -|---c3: New For Each(false,false)[bag] - scope-394 +|---c3: New For Each(false,false)[bag] - scope-392 | | - | Project[int][0] - scope-395 + | Project[int][0] - scope-393 | | - | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-396 + | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-394 | | - | |---Project[bag][0] - scope-397 + | |---Project[bag][0] - scope-395 | | - | |---Project[bag][1] - scope-398 + | |---Project[bag][1] - scope-396 | - |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-409 + |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-407 | - |---POValueInputTez - scope-340 <- scope-331 -Tez vertex scope-341 -# Combine plan on edge <scope-339> -c2: Local Rearrange[tuple]{int}(false) - scope-410 -> scope-341 + |---POValueInputTez - scope-339 <- scope-330 +Tez vertex scope-340 +# Combine plan on edge <scope-338> +c2: Local Rearrange[tuple]{int}(false) - scope-408 -> scope-340 | | -| Project[int][0] - scope-412 +| Project[int][0] - scope-410 | -|---c3: New For Each(false,false)[bag] - scope-399 +|---c3: New For Each(false,false)[bag] - scope-397 | | - | Project[int][0] - scope-400 + | Project[int][0] - scope-398 | | - | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-401 + | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-399 | | - | |---Project[bag][1] - scope-402 + | |---Project[bag][1] - scope-400 | - |---c2: Package(CombinerPackager)[tuple]{int} - scope-405 + |---c2: Package(CombinerPackager)[tuple]{int} - scope-403 # Plan on vertex -c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-280 +c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-279 | -|---c3: New For Each(false,false)[bag] - scope-279 +|---c3: New For Each(false,false)[bag] - scope-278 | | - | Project[int][0] - scope-273 + | Project[int][0] - scope-272 | | - | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-277 + | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-276 | | - | |---Project[bag][1] - scope-403 + | |---Project[bag][1] - scope-401 | - |---c2: Package(CombinerPackager)[tuple]{int} - scope-270 -Tez vertex scope-323 + |---c2: Package(CombinerPackager)[tuple]{int} - scope-269 +Tez vertex scope-322 # Plan on vertex -b1: Local Rearrange[tuple]{int}(false) - scope-230 -> scope-325 +b1: Local Rearrange[tuple]{int}(false) - scope-229 -> scope-324 | | -| Project[int][0] - scope-231 +| Project[int][0] - scope-230 | -|---POValueInputTez - scope-324 <- scope-320 -Tez vertex scope-325 +|---POValueInputTez - scope-323 <- scope-319 +Tez vertex scope-324 # Plan on vertex -POValueOutputTez - scope-326 -> [scope-329, scope-327] +POValueOutputTez - scope-325 -> [scope-328, scope-326] | -|---b1: Package(Packager)[tuple]{int} - scope-229 -Tez vertex scope-327 +|---b1: Package(Packager)[tuple]{int} - scope-228 +Tez vertex scope-326 # Plan on vertex -b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-235 +b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-234 | -|---POValueInputTez - scope-328 <- scope-325 -Tez vertex scope-345 +|---POValueInputTez - scope-327 <- scope-324 +Tez vertex scope-344 # Plan on vertex -d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-292 +d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-291 | -|---POValueInputTez - scope-346 <- scope-342 -Tez vertex scope-329 +|---POValueInputTez - scope-345 <- scope-341 +Tez vertex scope-328 # Plan on vertex -b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-245 +b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-244 | -|---b2: New For Each(false,false)[bag] - scope-244 +|---b2: New For Each(false,false)[bag] - scope-243 | | - | Project[int][0] - scope-238 + | Project[int][0] - scope-237 | | - | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-242 + | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-241 | | - | |---Project[bag][0] - scope-241 + | |---Project[bag][0] - scope-240 | | - | |---Project[bag][1] - scope-240 + | |---Project[bag][1] - scope-239 | - |---POValueInputTez - scope-330 <- scope-325 -Tez vertex scope-350 + |---POValueInputTez - scope-329 <- scope-324 +Tez vertex scope-349 # Plan on vertex -Local Rearrange[tuple]{tuple}(false) - scope-354 -> scope-360 +Local Rearrange[tuple]{tuple}(false) - scope-353 -> scope-359 | | -| Constant(DummyVal) - scope-353 +| Constant(DummyVal) - scope-352 | -|---New For Each(false,true)[tuple] - scope-359 +|---New For Each(false,true)[tuple] - scope-358 | | - | Project[int][0] - scope-300 + | Project[int][0] - scope-299 | | - | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-358 + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-357 | | - | |---Project[tuple][*] - scope-357 + | |---Project[tuple][*] - scope-356 | - |---ReservoirSample - scope-356 + |---ReservoirSample - scope-355 | - |---e1: Local Rearrange[tuple]{int}(false) - scope-352 -> scope-370 + |---e1: Local Rearrange[tuple]{int}(false) - scope-351 -> scope-369 | | - | Project[int][0] - scope-300 + | Project[int][0] - scope-299 | - |---e: Filter[bag] - scope-296 + |---e: Filter[bag] - scope-295 | | - | Less Than[boolean] - scope-299 + | Less Than[boolean] - scope-298 | | - | |---Project[int][0] - scope-297 + | |---Project[int][0] - scope-296 | | - | |---Constant(3) - scope-298 + | |---Constant(3) - scope-297 | - |---POValueInputTez - scope-351 <- scope-347 -Tez vertex scope-360 + |---POValueInputTez - scope-350 <- scope-346 +Tez vertex scope-359 # Plan on vertex -POValueOutputTez - scope-369 -> [scope-370] +POValueOutputTez - scope-368 -> [scope-369] | -|---New For Each(false)[tuple] - scope-368 +|---New For Each(false)[tuple] - scope-367 | | - | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-367 + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-366 | | - | |---Project[tuple][*] - scope-366 + | |---Project[tuple][*] - scope-365 | - |---New For Each(false,false)[tuple] - scope-365 + |---New For Each(false,false)[tuple] - scope-364 | | - | Constant(-1) - scope-364 + | Constant(-1) - scope-363 | | - | Project[bag][1] - scope-362 + | Project[bag][1] - scope-361 | - |---Package(Packager)[tuple]{bytearray} - scope-361 -Tez vertex scope-370 + |---Package(Packager)[tuple]{bytearray} - scope-360 +Tez vertex scope-369 # Plan on vertex -POIdentityInOutTez - scope-371 <- scope-350 -> scope-372 +POIdentityInOutTez - scope-370 <- scope-349 -> scope-371 | | -| Project[int][0] - scope-300 -Tez vertex scope-372 +| Project[int][0] - scope-299 +Tez vertex scope-371 # Plan on vertex -e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-302 +e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-301 | -|---New For Each(true)[tuple] - scope-375 +|---New For Each(true)[tuple] - scope-374 | | - | Project[bag][1] - scope-374 + | Project[bag][1] - scope-373 | - |---Package(LitePackager)[tuple]{int} - scope-373 -Tez vertex scope-334 + |---Package(LitePackager)[tuple]{int} - scope-372 +Tez vertex scope-333 # Plan on vertex -c1: Local Rearrange[tuple]{int}(false) - scope-259 -> scope-338 +c1: Local Rearrange[tuple]{int}(false) - scope-258 -> scope-337 | | -| Project[int][0] - scope-260 +| Project[int][0] - scope-259 | -|---POValueInputTez - scope-335 <- scope-331 -Tez vertex scope-336 +|---POValueInputTez - scope-334 <- scope-330 +Tez vertex scope-335 # Plan on vertex -c1: Local Rearrange[tuple]{int}(false) - scope-261 -> scope-338 +c1: Local Rearrange[tuple]{int}(false) - scope-260 -> scope-337 | | -| Project[int][0] - scope-262 +| Project[int][0] - scope-261 | -|---POValueInputTez - scope-337 <- scope-320 -Tez vertex scope-338 +|---POValueInputTez - scope-336 <- scope-319 +Tez vertex scope-337 # Plan on vertex -c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-266 +c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-265 | -|---c1: New For Each(true,true)[tuple] - scope-265 +|---c1: New For Each(true,true)[tuple] - scope-264 | | - | Project[bag][1] - scope-263 + | Project[bag][1] - scope-262 | | - | Project[bag][2] - scope-264 + | Project[bag][2] - scope-263 | - |---c1: Package(Packager)[tuple]{int} - scope-258 + |---c1: Package(Packager)[tuple]{int} - scope-257 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Thu Nov 27 12:49:54 2014 @@ -2,11 +2,11 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: scope-181 +# TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-167,Tez vertex group scope-209, -Tez vertex scope-167 -> Tez vertex group scope-209, -Tez vertex group scope-209 +Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-167,Tez vertex group scope-208, +Tez vertex scope-167 -> Tez vertex group scope-208, +Tez vertex group scope-208 Tez vertex scope-113 Tez vertex scope-148 -> Tez vertex scope-158, Tez vertex scope-158 -> Tez vertex scope-160, @@ -17,9 +17,9 @@ Tez vertex scope-129 Tez vertex scope-106 # Plan on vertex -1-1: Split - scope-208 +1-1: Split - scope-207 | | -| b: Split - scope-205 +| b: Split - scope-204 | | | | | b1: Local Rearrange[tuple]{int}(false) - scope-18 -> scope-113 | | | | @@ -29,7 +29,7 @@ Tez vertex scope-106 | | | | | | | Project[int][0] - scope-50 | | | -| | 1-2: Split - scope-204 +| | 1-2: Split - scope-203 | | | | | | | Local Rearrange[tuple]{tuple}(false) - scope-142 -> scope-148 | | | | | @@ -77,11 +77,11 @@ Tez vertex scope-106 | | | | | |---Constant(5) - scope-11 | | -| d1: Split - scope-207 +| d1: Split - scope-206 | | | | | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80 | | | -| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 +| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 | | | |---d1: Filter[bag] - scope-73 | | | @@ -114,20 +114,20 @@ Tez vertex scope-106 |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-167 # Plan on vertex -f1: Split - scope-203 +f1: Split - scope-202 | | | f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99 | | -| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-211 +| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 | |---f1: Limit - scope-169 | |---POValueInputTez - scope-168 <- scope-106 -Tez vertex group scope-209 <- [scope-106, scope-167] -> null +Tez vertex group scope-208 <- [scope-106, scope-167] -> null # No plan on vertex group Tez vertex scope-113 # Plan on vertex -b1: Split - scope-202 +b1: Split - scope-201 | | | b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23 | | @@ -150,7 +150,7 @@ POValueOutputTez - scope-157 -> [scope- | |---New For Each(false)[tuple] - scope-156 | | - | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-155 + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-155 | | | |---Project[tuple][*] - scope-154 | @@ -177,27 +177,27 @@ e1: Store(file:///tmp/output/e1:org.apac |---Package(LitePackager)[tuple]{int} - scope-161 Tez vertex scope-119 # Plan on vertex -c: Split - scope-206 +c: Split - scope-205 | | | c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126 | | | | | Project[int][0] - scope-48 | | -| c2: Local Rearrange[tuple]{int}(false) - scope-194 -> scope-129 +| c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129 | | | -| | Project[int][0] - scope-196 +| | Project[int][0] - scope-195 | | -| |---c3: New For Each(false,false)[bag] - scope-182 +| |---c3: New For Each(false,false)[bag] - scope-181 | | | -| | Project[int][0] - scope-183 +| | Project[int][0] - scope-182 | | | -| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-184 +| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183 | | | -| | |---Project[bag][0] - scope-185 +| | |---Project[bag][0] - scope-184 | | | -| | |---Project[bag][1] - scope-186 +| | |---Project[bag][1] - scope-185 | | -| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-197 +| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196 | |---c: Filter[bag] - scope-34 | | @@ -221,19 +221,19 @@ c1: Store(file:///tmp/output/c1:org.apac |---c1: Package(Packager)[tuple]{int} - scope-46 Tez vertex scope-129 # Combine plan on edge <scope-119> -c2: Local Rearrange[tuple]{int}(false) - scope-198 -> scope-129 +c2: Local Rearrange[tuple]{int}(false) - scope-197 -> scope-129 | | -| Project[int][0] - scope-200 +| Project[int][0] - scope-199 | -|---c3: New For Each(false,false)[bag] - scope-187 +|---c3: New For Each(false,false)[bag] - scope-186 | | - | Project[int][0] - scope-188 + | Project[int][0] - scope-187 | | - | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-189 + | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-188 | | - | |---Project[bag][1] - scope-190 + | |---Project[bag][1] - scope-189 | - |---c2: Package(CombinerPackager)[tuple]{int} - scope-193 + |---c2: Package(CombinerPackager)[tuple]{int} - scope-192 # Plan on vertex c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68 | @@ -243,6 +243,6 @@ c3: Store(file:///tmp/output/c1:org.apac | | | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-65 | | - | |---Project[bag][1] - scope-191 + | |---Project[bag][1] - scope-190 | |---c2: Package(CombinerPackager)[tuple]{int} - scope-58 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld Thu Nov 27 12:49:54 2014 @@ -2,118 +2,118 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: scope-134 +# TEZ DAG plan: pig-0_scope-1 #-------------------------------------------------- -Tez vertex scope-126 -> Tez vertex scope-128,Tez vertex scope-131, -Tez vertex scope-131 -> Tez vertex scope-133, -Tez vertex scope-133 -Tez vertex scope-128 -> Tez vertex scope-130, -Tez vertex scope-130 +Tez vertex scope-125 -> Tez vertex scope-127,Tez vertex scope-130, +Tez vertex scope-127 -> Tez vertex scope-129, +Tez vertex scope-129 +Tez vertex scope-130 -> Tez vertex scope-132, +Tez vertex scope-132 -Tez vertex scope-126 +Tez vertex scope-125 # Plan on vertex -POValueOutputTez - scope-127 -> [scope-128, scope-131] +POValueOutputTez - scope-126 -> [scope-127, scope-130] | -|---a: New For Each(false,false)[bag] - scope-95 +|---a: New For Each(false,false)[bag] - scope-94 | | - | Cast[int] - scope-90 + | Cast[int] - scope-89 | | - | |---Project[bytearray][0] - scope-89 + | |---Project[bytearray][0] - scope-88 | | - | Cast[int] - scope-93 + | Cast[int] - scope-92 | | - | |---Project[bytearray][1] - scope-92 + | |---Project[bytearray][1] - scope-91 | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-88 -Tez vertex scope-131 + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-87 +Tez vertex scope-127 # Plan on vertex -c: Local Rearrange[tuple]{tuple}(false) - scope-166 -> scope-133 +b: Local Rearrange[tuple]{int}(false) - scope-145 -> scope-129 | | -| Project[tuple][0] - scope-169 +| Project[int][0] - scope-147 | -|---c: New For Each(false,false)[bag] - scope-154 +|---b: New For Each(false,false)[bag] - scope-133 | | - | Project[tuple][0] - scope-155 + | Project[int][0] - scope-134 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-156 + | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-135 | | - | |---Project[bag][1] - scope-157 + | |---Project[bag][0] - scope-136 | | - | |---Project[bag][1] - scope-158 + | |---Project[bag][1] - scope-137 | - |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-170 + |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-148 | - |---POValueInputTez - scope-132 <- scope-126 -Tez vertex scope-133 -# Combine plan on edge <scope-131> -c: Local Rearrange[tuple]{tuple}(false) - scope-171 -> scope-133 + |---POValueInputTez - scope-128 <- scope-125 +Tez vertex scope-129 +# Combine plan on edge <scope-127> +b: Local Rearrange[tuple]{int}(false) - scope-149 -> scope-129 | | -| Project[tuple][0] - scope-174 +| Project[int][0] - scope-151 | -|---c: New For Each(false,false)[bag] - scope-159 +|---b: New For Each(false,false)[bag] - scope-138 | | - | Project[tuple][0] - scope-160 + | Project[int][0] - scope-139 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-161 + | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-140 | | - | |---Project[bag][1] - scope-162 + | |---Project[bag][1] - scope-141 | - |---c: Package(CombinerPackager)[tuple]{tuple} - scope-165 + |---b: Package(CombinerPackager)[tuple]{int} - scope-144 # Plan on vertex -c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-125 +b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-109 | -|---c: New For Each(false,false)[bag] - scope-124 +|---b: New For Each(false,false)[bag] - scope-108 | | - | Project[tuple][0] - scope-118 + | Project[int][0] - scope-102 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-122 + | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-106 | | - | |---Project[bag][1] - scope-163 + | |---Project[bag][1] - scope-142 | - |---c: Package(CombinerPackager)[tuple]{tuple} - scope-114 -Tez vertex scope-128 + |---b: Package(CombinerPackager)[tuple]{int} - scope-99 +Tez vertex scope-130 # Plan on vertex -b: Local Rearrange[tuple]{int}(false) - scope-147 -> scope-130 +c: Local Rearrange[tuple]{tuple}(false) - scope-164 -> scope-132 | | -| Project[int][0] - scope-149 +| Project[tuple][0] - scope-167 | -|---b: New For Each(false,false)[bag] - scope-135 +|---c: New For Each(false,false)[bag] - scope-152 | | - | Project[int][0] - scope-136 + | Project[tuple][0] - scope-153 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-137 + | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-154 | | - | |---Project[bag][0] - scope-138 + | |---Project[bag][1] - scope-155 | | - | |---Project[bag][1] - scope-139 + | |---Project[bag][1] - scope-156 | - |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-150 + |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-168 | - |---POValueInputTez - scope-129 <- scope-126 -Tez vertex scope-130 -# Combine plan on edge <scope-128> -b: Local Rearrange[tuple]{int}(false) - scope-151 -> scope-130 + |---POValueInputTez - scope-131 <- scope-125 +Tez vertex scope-132 +# Combine plan on edge <scope-130> +c: Local Rearrange[tuple]{tuple}(false) - scope-169 -> scope-132 | | -| Project[int][0] - scope-153 +| Project[tuple][0] - scope-172 | -|---b: New For Each(false,false)[bag] - scope-140 +|---c: New For Each(false,false)[bag] - scope-157 | | - | Project[int][0] - scope-141 + | Project[tuple][0] - scope-158 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-142 + | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-159 | | - | |---Project[bag][1] - scope-143 + | |---Project[bag][1] - scope-160 | - |---b: Package(CombinerPackager)[tuple]{int} - scope-146 + |---c: Package(CombinerPackager)[tuple]{tuple} - scope-163 # Plan on vertex -b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-110 +c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-124 | -|---b: New For Each(false,false)[bag] - scope-109 +|---c: New For Each(false,false)[bag] - scope-123 | | - | Project[int][0] - scope-103 + | Project[tuple][0] - scope-117 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-107 + | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-121 | | - | |---Project[bag][1] - scope-144 + | |---Project[bag][1] - scope-161 | - |---b: Package(CombinerPackager)[tuple]{int} - scope-100 + |---c: Package(CombinerPackager)[tuple]{tuple} - scope-113 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld Thu Nov 27 12:49:54 2014 @@ -2,7 +2,7 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: scope-46 +# TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- Tez vertex scope-38 -> Tez vertex scope-42,Tez vertex scope-45, Tez vertex scope-45 @@ -10,39 +10,39 @@ Tez vertex scope-42 Tez vertex scope-38 # Plan on vertex -a: Split - scope-87 +a: Split - scope-86 | | -| b: Local Rearrange[tuple]{int}(false) - scope-59 -> scope-42 +| b: Local Rearrange[tuple]{int}(false) - scope-58 -> scope-42 | | | -| | Project[int][0] - scope-61 +| | Project[int][0] - scope-60 | | -| |---b: New For Each(false,false)[bag] - scope-47 +| |---b: New For Each(false,false)[bag] - scope-46 | | | -| | Project[int][0] - scope-48 +| | Project[int][0] - scope-47 | | | -| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-49 +| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-48 | | | -| | |---Project[bag][0] - scope-50 +| | |---Project[bag][0] - scope-49 | | | -| | |---Project[bag][1] - scope-51 +| | |---Project[bag][1] - scope-50 | | -| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-62 +| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-61 | | -| c: Local Rearrange[tuple]{tuple}(false) - scope-78 -> scope-45 +| c: Local Rearrange[tuple]{tuple}(false) - scope-77 -> scope-45 | | | -| | Project[tuple][0] - scope-81 +| | Project[tuple][0] - scope-80 | | -| |---c: New For Each(false,false)[bag] - scope-66 +| |---c: New For Each(false,false)[bag] - scope-65 | | | -| | Project[tuple][0] - scope-67 +| | Project[tuple][0] - scope-66 | | | -| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-68 +| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-67 | | | -| | |---Project[bag][1] - scope-69 +| | |---Project[bag][1] - scope-68 | | | -| | |---Project[bag][1] - scope-70 +| | |---Project[bag][1] - scope-69 | | -| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-82 +| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-81 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -57,19 +57,19 @@ a: Split - scope-87 |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-45 # Combine plan on edge <scope-38> -c: Local Rearrange[tuple]{tuple}(false) - scope-83 -> scope-45 +c: Local Rearrange[tuple]{tuple}(false) - scope-82 -> scope-45 | | -| Project[tuple][0] - scope-86 +| Project[tuple][0] - scope-85 | -|---c: New For Each(false,false)[bag] - scope-71 +|---c: New For Each(false,false)[bag] - scope-70 | | - | Project[tuple][0] - scope-72 + | Project[tuple][0] - scope-71 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-73 + | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-72 | | - | |---Project[bag][1] - scope-74 + | |---Project[bag][1] - scope-73 | - |---c: Package(CombinerPackager)[tuple]{tuple} - scope-77 + |---c: Package(CombinerPackager)[tuple]{tuple} - scope-76 # Plan on vertex c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-37 | @@ -79,24 +79,24 @@ c: Store(file:///tmp/output/c:org.apache | | | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-34 | | - | |---Project[bag][1] - scope-75 + | |---Project[bag][1] - scope-74 | |---c: Package(CombinerPackager)[tuple]{tuple} - scope-26 Tez vertex scope-42 # Combine plan on edge <scope-38> -b: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-42 +b: Local Rearrange[tuple]{int}(false) - scope-62 -> scope-42 | | -| Project[int][0] - scope-65 +| Project[int][0] - scope-64 | -|---b: New For Each(false,false)[bag] - scope-52 +|---b: New For Each(false,false)[bag] - scope-51 | | - | Project[int][0] - scope-53 + | Project[int][0] - scope-52 | | - | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-54 + | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-53 | | - | |---Project[bag][1] - scope-55 + | |---Project[bag][1] - scope-54 | - |---b: Package(CombinerPackager)[tuple]{int} - scope-58 + |---b: Package(CombinerPackager)[tuple]{int} - scope-57 # Plan on vertex b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-22 | @@ -106,6 +106,6 @@ b: Store(file:///tmp/output/b:org.apache | | | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-19 | | - | |---Project[bag][1] - scope-56 + | |---Project[bag][1] - scope-55 | |---b: Package(CombinerPackager)[tuple]{int} - scope-12
