Modified: pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java Fri Mar 4 18:17:39 2016 @@ -51,7 +51,6 @@ import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.impl.builtin.PartitionSkewedKeys; import org.apache.pig.impl.logicalLayer.FrontendException; -import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor; import org.apache.pig.test.utils.TestHelper; import org.junit.AfterClass; import org.junit.Before; @@ -66,8 +65,9 @@ public class TestSkewedJoin { private static final String INPUT_FILE5 = "SkewedJoinInput5.txt"; private static final String INPUT_FILE6 = "SkewedJoinInput6.txt"; private static final String INPUT_FILE7 = "SkewedJoinInput7.txt"; - private static final String INPUT_DIR = "build/test/data"; - private static final String OUTPUT_DIR = "build/test/output"; + private static final String TEST_DIR = Util.getTestDirectory(TestSkewedJoin.class); + private static final String INPUT_DIR = TEST_DIR + Path.SEPARATOR + "input"; + private static final String OUTPUT_DIR = TEST_DIR + Path.SEPARATOR + "output"; private static FileSystem fs; private static PigServer pigServer; @@ -97,8 +97,8 @@ public class TestSkewedJoin { } private static void createFiles() throws IOException { - new File(INPUT_DIR).mkdir(); - new File(OUTPUT_DIR).mkdir(); + new File(INPUT_DIR).mkdirs(); + new File(OUTPUT_DIR).mkdirs(); PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1)); @@ -183,8 +183,7 @@ public class TestSkewedJoin { } private static void deleteFiles() throws IOException { - Util.deleteDirectory(new File(INPUT_DIR)); - Util.deleteDirectory(new File(OUTPUT_DIR)); + Util.deleteDirectory(new File(TEST_DIR)); } @Test @@ -563,7 +562,7 @@ public class TestSkewedJoin { SimpleLayout layout = new SimpleLayout(); FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0); logger.addAppender(appender); - + try { pigServer.registerScript(new ByteArrayInputStream(script.getBytes("UTF-8"))); pigServer.openIterator("joined");
Modified: pig/branches/spark/test/org/apache/pig/test/TestStreaming.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreaming.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStreaming.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStreaming.java Fri Mar 4 18:17:39 2016 @@ -40,14 +40,23 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class TestStreaming { private static final MiniGenericCluster cluster = MiniGenericCluster.buildCluster(); + private static File testDataDir; private PigServer pigServer; + @BeforeClass + public static void oneTimeSetUp() throws Exception { + // Create the test data directory if needed + testDataDir = new File(Util.getTestDirectory(TestStreaming.class)); + testDataDir.mkdirs(); + } + @Before public void setup() throws ExecException { pigServer = new PigServer(cluster.getExecType(), cluster.getProperties()); @@ -61,6 +70,7 @@ public class TestStreaming { @AfterClass public static void oneTimeTearDown() throws Exception { cluster.shutDown(); + Util.deleteDirectory(testDataDir); } private TupleFactory tf = TupleFactory.getInstance(); @@ -293,7 +303,9 @@ public class TestStreaming { "}", }; File command1 = Util.createInputFile("script", "pl", script); - File command2 = Util.createInputFile("script", "pl", script); + // Test relative path + File command2 = new File(testDataDir, "testInputShipSpecs.pl"); + Util.writeToFile(command2, script); // Expected results String[] expectedFirstFields = Modified: pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java Fri Mar 4 18:17:39 2016 @@ -128,7 +128,10 @@ public class TestStringUDFs { REPLACE replace = new REPLACE(); Tuple testTuple = Util.buildTuple("foobar", "z", "x"); assertEquals("foobar".replace("z", "x"), replace.exec(testTuple)); - + + // Use cached version of pattern in REPLACE + assertEquals("foobar".replace("z", "x"), replace.exec(testTuple)); + testTuple = Util.buildTuple("foobar", "oo", "aa"); assertEquals("foobar".replace("oo", "aa"), replace.exec(testTuple)); } Modified: pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Fri Mar 4 18:17:39 2016 @@ -745,8 +745,8 @@ public class TestTypeCheckingValidatorNe // value of datetime and one of other type LogicalExpressionPlan plan = new LogicalExpressionPlan(); ConstantExpression constant0 = new ConstantExpression(plan, new DateTime(0L)); - ConstantExpression constant1 = new ConstantExpression(plan, new DataByteArray("1970-01-01T00:00:00.000Z")); - CastExpression cast1 = new CastExpression(plan, constant1, createFS(DataType.BYTEARRAY)); + ConstantExpression constant1 = new ConstantExpression(plan, new String("1970-01-01T00:00:00.000Z")); + CastExpression cast1 = new CastExpression(plan, constant1, createFS(DataType.CHARARRAY)); EqualExpression eq1 = new EqualExpression(plan, constant0, cast1); CompilationMessageCollector collector = new CompilationMessageCollector(); @@ -3685,6 +3685,7 @@ public class TestTypeCheckingValidatorNe super(p, walker); } + @Override public void visit(CastExpression cExp){ casts.add(cExp); } Modified: pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java Fri Mar 4 18:17:39 2016 @@ -17,28 +17,44 @@ */ package org.apache.pig.test; +import static org.apache.pig.builtin.mock.Storage.resetData; +import static org.apache.pig.builtin.mock.Storage.tuple; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileWriter; +import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Properties; +import org.apache.hadoop.mapreduce.Job; import org.apache.pig.PigServer; +import org.apache.pig.ResourceSchema; import org.apache.pig.builtin.PigStorage; +import org.apache.pig.builtin.mock.Storage; +import org.apache.pig.data.DataByteArray; import org.apache.pig.data.Tuple; import org.apache.pig.impl.io.FileLocalizer; +import org.apache.pig.impl.util.ObjectSerializer; import org.apache.pig.impl.util.UDFContext; +import org.junit.AfterClass; import org.junit.Test; +import com.google.common.io.Files; + public class TestUDFContext { + @AfterClass + public static void oneTimeTearDown() throws Exception { + FileLocalizer.deleteTempFiles(); + } + @Test public void testUDFContext() throws Exception { - File a = Util.createLocalInputFile("a.txt", new String[] { "dumb" }); - File b = Util.createLocalInputFile("b.txt", new String[] { "dumber" }); - FileLocalizer.deleteTempFiles(); + File a = Util.createInputFile("inp1", "txt", new String[] { "dumb" }); + File b = Util.createInputFile("inp2", "txt", new String[] { "dumber" }); PigServer pig = new PigServer(Util.getLocalTestMode(), new Properties()); String[] statement = { "A = LOAD '" + Util.encodeEscape(a.getAbsolutePath()) + "' USING org.apache.pig.test.utils.UDFContextTestLoader('joe');", @@ -100,5 +116,112 @@ public class TestUDFContext { } + + @Test + public void testUDFContextSeparator() throws Exception { + + File inputFile = Util.createInputFile("input", "txt", new String[] { "f1\tf2\tf3\tf4\tf5" }); + + PigServer pigServer = new PigServer(Util.getLocalTestMode(), new Properties()); + Storage.Data data = resetData(pigServer); + + String inputPath = Util.encodeEscape(inputFile.getAbsolutePath()); + File file = Files.createTempDir(); + file.deleteOnExit(); + String outputFile = Util.generateURI(file.getPath() + File.pathSeparator + "out", pigServer.getPigContext()); + String query = "A = LOAD '" + inputPath + "' USING PigStorage() as (f1, f2, f3, f4, f5);" + + "B = LOAD '" + inputPath + "' USING PigStorage();" + + "B = FOREACH B GENERATE $0, $1;" + + "C = LOAD '" + inputPath + "' USING " + FieldsByIndexLoader.class.getName() + "('1,2');" + // Scalar to force PigStorage to be always visited first in LoaderProcessor + + "C = FOREACH C GENERATE *, B.$0;" + + "STORE A INTO 'A' USING mock.Storage();" + + "STORE B INTO 'B' USING mock.Storage();" + + "STORE C INTO 'C' USING mock.Storage();" + + "STORE A INTO '" + outputFile + "' USING " + FieldsByIndexLoader.class.getName() + "();"; + + pigServer.registerQuery(query); + + List<Tuple> a = data.get("A"); + List<Tuple> b = data.get("B"); + List<Tuple> c = data.get("C"); + assertEquals(1, a.size()); + assertEquals(1, b.size()); + assertEquals(1, c.size()); + DataByteArray f1 = new DataByteArray("f1"); + DataByteArray f2 = new DataByteArray("f2"); + DataByteArray f3 = new DataByteArray("f3"); + DataByteArray f4 = new DataByteArray("f4"); + DataByteArray f5 = new DataByteArray("f5"); + assertEquals(tuple(f1, f2, f3, f4, f5), a.get(0)); + assertEquals(tuple(f1, f2), b.get(0)); + assertEquals(tuple(f2, f3, f1), c.get(0)); + } + + + public static class FieldsByIndexLoader extends PigStorage { + + // Tests the case of one user LoadFunc where UDF properties was a class variable + // and getSchema() was used to determined frontend instead of + // UDFContext.getUDFContext().isFrontEnd(); + private boolean frontend = false; + private Properties props = UDFContext.getUDFContext().getUDFProperties(this.getClass()); + private boolean[] selectedFields = new boolean[5]; //Assuming data always has 5 columns + private String storeSignature; + + public FieldsByIndexLoader() { + } + + public FieldsByIndexLoader(String fieldIndices) { + String[] requiredFields = fieldIndices.split(","); + for (String index : requiredFields) { + selectedFields[Integer.parseInt(index)] = true; + } + } + + @Override + public void setLocation(String location, Job job) throws IOException { + if (frontend) { + // PigStorage should deserialize this as + // mRequiredColumns = (boolean[])ObjectSerializer.deserialize(p.getProperty(signature)); + props.setProperty(signature, ObjectSerializer.serialize(selectedFields)); + } + super.setLocation(location, job); + } + + @Override + public ResourceSchema getSchema(String location, Job job) + throws IOException { + frontend = true; + return super.getSchema(location, job); + } + + @Override + public void checkSchema(ResourceSchema s) throws IOException { + super.checkSchema(s); + UDFContext udfContext = UDFContext.getUDFContext(); + Properties props = udfContext.getUDFProperties(this.getClass(), new String[]{storeSignature}); + props.setProperty("testkey", "testvalue"); + } + + @Override + public void setStoreFuncUDFContextSignature(String signature) { + this.storeSignature = signature; + } + + @Override + public void setStoreLocation(String location, Job job) + throws IOException { + if (!UDFContext.getUDFContext().isFrontend()) { + Properties udfProps = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[]{storeSignature}); + if (!("testvalue").equals(udfProps.getProperty("testkey"))) { + throw new IOException("UDFContext does not have expected values"); + } + } + super.setStoreLocation(location, job); + } + + + } } Modified: pig/branches/spark/test/org/apache/pig/test/Util.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/Util.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/Util.java (original) +++ pig/branches/spark/test/org/apache/pig/test/Util.java Fri Mar 4 18:17:39 2016 @@ -118,6 +118,8 @@ public class Util { public static final boolean WINDOWS /* borrowed from Path.WINDOWS, Shell.WINDOWS */ = System.getProperty("os.name").startsWith("Windows"); + public static final String TEST_DIR = System.getProperty("test.build.dir", "build/test"); + // Helper Functions // ================= @@ -176,6 +178,28 @@ public class Util { return t; } + /** + * Create an array of tuple bags with specified size created by splitting + * the input array of primitive types + * + * @param input Array of primitive types + * @param bagSize The number of tuples to be split and copied into each bag + * + * @return an array of tuple bags with each bag containing bagSize tuples split from the input + */ + static public <T> Tuple[] splitCreateBagOfTuples(T[] input, int bagSize) + throws ExecException { + List<Tuple> result = new ArrayList<Tuple>(); + for (int from = 0; from < input.length; from += bagSize) { + Tuple t = TupleFactory.getInstance().newTuple(1); + int to = from + bagSize < input.length ? from + bagSize + : input.length; + T[] array = Arrays.copyOfRange(input, from, to); + result.add(loadNestTuple(t, array)); + } + return result.toArray(new Tuple[0]); + } + static public <T>void addToTuple(Tuple t, T[] b) { for(int i = 0; i < b.length; i++) @@ -934,6 +958,7 @@ public class Util { thread = new Thread (this); thread.start (); } + @Override public void run () { try { InputStreamReader isr = new InputStreamReader (is); @@ -1392,37 +1417,47 @@ public class Util { return ExecTypeProvider.fromString("local"); } - public static void createLogAppender(Class clazz, String appenderName, Writer writer) { - Logger logger = Logger.getLogger(clazz); + public static void createLogAppender(String appenderName, Writer writer, Class...clazzes) { WriterAppender writerAppender = new WriterAppender(new PatternLayout("%d [%t] %-5p %c %x - %m%n"), writer); writerAppender.setName(appenderName); - logger.addAppender(writerAppender); + for (Class clazz : clazzes) { + Logger logger = Logger.getLogger(clazz); + logger.addAppender(writerAppender); + } } - public static void removeLogAppender(Class clazz, String appenderName) { - Logger logger = Logger.getLogger(clazz); - Appender appender = logger.getAppender(appenderName); - appender.close(); - logger.removeAppender(appenderName); + public static void removeLogAppender(String appenderName, Class...clazzes) { + for (Class clazz : clazzes) { + Logger logger = Logger.getLogger(clazz); + Appender appender = logger.getAppender(appenderName); + appender.close(); + logger.removeAppender(appenderName); + } } public static Path getFirstPartFile(Path path) throws Exception { - FileStatus[] parts = FileSystem.get(path.toUri(), new Configuration()).listStatus(path, + FileStatus[] parts = FileSystem.get(path.toUri(), new Configuration()).listStatus(path, new PathFilter() { @Override public boolean accept(Path path) { return path.getName().startsWith("part-"); } }); - return parts[0].getPath(); + return parts[0].getPath(); } public static File getFirstPartFile(File dir) throws Exception { File[] parts = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { return name.startsWith("part-"); }; }); - return parts[0]; + return parts[0]; + } + + @SuppressWarnings("rawtypes") + public static String getTestDirectory(Class testClass) { + return TEST_DIR + Path.SEPARATOR + "testdata" + Path.SEPARATOR +testClass.getSimpleName(); } } Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld Fri Mar 4 18:17:39 2016 @@ -4,86 +4,92 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-12 -> Tez vertex scope-22,Tez vertex scope-33, -Tez vertex scope-22 -> Tez vertex scope-33, -Tez vertex scope-33 -> Tez vertex scope-35, -Tez vertex scope-35 -> Tez vertex scope-46, -Tez vertex scope-46 +Tez vertex scope-12 -> Tez vertex scope-28, +Tez vertex scope-28 -> Tez vertex scope-39, +Tez vertex scope-39 -> Tez vertex scope-40, +Tez vertex scope-40 -> Tez vertex scope-51, +Tez vertex scope-51 Tez vertex scope-12 # Plan on vertex -Local Rearrange[tuple]{tuple}(false) - scope-15 -> scope-22 +Local Rearrange[tuple]{tuple}(false) - scope-21 -> scope-28 | | -| Constant(DummyVal) - scope-14 +| Constant(DummyVal) - scope-20 | -|---New For Each(false,false,true)[tuple] - scope-21 +|---New For Each(false,false,true)[tuple] - scope-27 | | | Project[int][0] - scope-8 | | | Project[int][1] - scope-9 | | - | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-20 + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-26 | | - | |---Project[tuple][*] - scope-19 + | |---Project[tuple][*] - scope-25 | - |---ReservoirSample - scope-18 + |---ReservoirSample - scope-24 | - |---b: Local Rearrange[tuple]{tuple}(false) - scope-13 -> scope-33 + |---a: New For Each(false,false)[bag] - scope-7 | | - | Project[int][0] - scope-8 + | Cast[int] - scope-2 | | - | Project[int][1] - scope-9 + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 | - |---a: New For Each(false,false)[bag] - scope-7 - | | - | Cast[int] - scope-2 - | | - | |---Project[bytearray][0] - scope-1 - | | - | Cast[int] - scope-5 - | | - | |---Project[bytearray][1] - scope-4 - | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-22 + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-28 # Plan on vertex -POValueOutputTez - scope-32 -> [scope-33] +POValueOutputTez - scope-38 -> [scope-39] | -|---New For Each(false)[tuple] - scope-31 +|---New For Each(false)[tuple] - scope-37 | | - | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-30 + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-36 | | - | |---Project[tuple][*] - scope-29 + | |---Project[tuple][*] - scope-35 | - |---New For Each(false,false)[tuple] - scope-28 + |---New For Each(false,false)[tuple] - scope-34 | | - | Constant(-1) - scope-27 + | Constant(-1) - scope-33 | | - | Project[bag][1] - scope-24 + | Project[bag][1] - scope-30 | - |---Package(Packager)[tuple]{bytearray} - scope-23 -Tez vertex scope-33 + |---Package(Packager)[tuple]{bytearray} - scope-29 +Tez vertex scope-39 # Plan on vertex -POIdentityInOutTez - scope-34 <- scope-12 -> scope-35 +b: Local Rearrange[tuple]{tuple}(false) - scope-13 -> scope-40 | | | Project[int][0] - scope-8 | | | Project[int][1] - scope-9 -Tez vertex scope-35 +| +|---a: New For Each(false,false)[bag] - scope-19 + | | + | Cast[int] - scope-16 + | | + | |---Project[bytearray][0] - scope-15 + | | + | Cast[int] - scope-18 + | | + | |---Project[bytearray][1] - scope-17 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-14 +Tez vertex scope-40 # Plan on vertex -POValueOutputTez - scope-45 -> [scope-46] +POValueOutputTez - scope-50 -> [scope-51] | -|---Limit - scope-44 +|---Limit - scope-49 | - |---New For Each(true)[tuple] - scope-43 + |---New For Each(true)[tuple] - scope-48 | | - | Project[bag][1] - scope-42 + | Project[bag][1] - scope-47 | - |---Package(LitePackager)[tuple]{tuple} - scope-41 -Tez vertex scope-46 + |---Package(LitePackager)[tuple]{tuple} - scope-46 +Tez vertex scope-51 # Plan on vertex c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-11 | -|---Limit - scope-48 +|---Limit - scope-53 | - |---POValueInputTez - scope-47 <- scope-35 + |---POValueInputTez - scope-52 <- scope-40 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Fri Mar 4 18:17:39 2016 @@ -294,12 +294,12 @@ f1: Store(file:///tmp/output/f1:org.apac |---POValueInputTez - scope-383 <- scope-378 Tez vertex scope-384 # Plan on vertex -f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 -> scope-316 | |---POValueInputTez - scope-385 <- scope-341 Tez vertex scope-386 # Plan on vertex -f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 -> scope-316 | |---POValueInputTez - scope-387 <- scope-378 Tez vertex group scope-412 <- [scope-384, scope-386] -> null Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Fri Mar 4 18:17:39 2016 @@ -4,9 +4,8 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-106 -> Tez vertex scope-113,Tez vertex scope-119,Tez vertex scope-126,Tez vertex scope-148,Tez vertex scope-158,Tez vertex scope-167,Tez vertex group scope-208, +Tez vertex scope-106 -> Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-129,Tez vertex scope-148,Tez vertex scope-158,Tez vertex scope-167,Tez vertex group scope-208, Tez vertex scope-113 -Tez vertex scope-119 -> Tez vertex scope-126,Tez vertex scope-129, Tez vertex scope-126 Tez vertex scope-129 Tez vertex scope-148 -> Tez vertex scope-158, @@ -77,11 +76,41 @@ Tez vertex scope-106 | | | | | |---Constant(5) - scope-11 | | +| c: Split - scope-205 +| | | +| | c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126 +| | | | +| | | Project[int][0] - scope-48 +| | | +| | c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129 +| | | | +| | | Project[int][0] - scope-195 +| | | +| | |---c3: New For Each(false,false)[bag] - scope-181 +| | | | +| | | Project[int][0] - scope-182 +| | | | +| | | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183 +| | | | +| | | |---Project[bag][0] - scope-184 +| | | | +| | | |---Project[bag][1] - scope-185 +| | | +| | |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196 +| | +| |---c: Filter[bag] - scope-34 +| | | +| | Less Than or Equal[boolean] - scope-37 +| | | +| | |---Project[int][0] - scope-35 +| | | +| | |---Constant(10) - scope-36 +| | | d1: Split - scope-206 | | | | | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80 | | | -| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 +| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 -> scope-105 | | | |---d1: Filter[bag] - scope-73 | | | @@ -98,8 +127,6 @@ Tez vertex scope-106 | | |---Project[int][0] - scope-70 | | | | | |---Constant(10) - scope-71 -| | -| POValueOutputTez - scope-107 -> [scope-119] | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -131,39 +158,6 @@ b1: Split - scope-201 | | |---Project[bag][1] - scope-28 | |---b1: Package(Packager)[tuple]{int} - scope-17 -Tez vertex scope-119 -# Plan on vertex -c: Split - scope-205 -| | -| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126 -| | | -| | Project[int][0] - scope-48 -| | -| c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129 -| | | -| | Project[int][0] - scope-195 -| | -| |---c3: New For Each(false,false)[bag] - scope-181 -| | | -| | Project[int][0] - scope-182 -| | | -| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183 -| | | -| | |---Project[bag][0] - scope-184 -| | | -| | |---Project[bag][1] - scope-185 -| | -| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196 -| -|---c: Filter[bag] - scope-34 - | | - | Less Than or Equal[boolean] - scope-37 - | | - | |---Project[int][0] - scope-35 - | | - | |---Constant(10) - scope-36 - | - |---POValueInputTez - scope-120 <- scope-106 Tez vertex scope-126 # Plan on vertex c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-54 @@ -176,7 +170,7 @@ c1: Store(file:///tmp/output/c1:org.apac | |---c1: Package(Packager)[tuple]{int} - scope-46 Tez vertex scope-129 -# Combine plan on edge <scope-119> +# Combine plan on edge <scope-106> c2: Local Rearrange[tuple]{int}(false) - scope-197 -> scope-129 | | | Project[int][0] - scope-199 @@ -239,7 +233,7 @@ f1: Split - scope-202 | | | f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99 | | -| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 +| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 -> scope-105 | |---f1: Limit - scope-169 | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld Fri Mar 4 18:17:39 2016 @@ -4,70 +4,78 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-11 -> Tez vertex scope-20,Tez vertex scope-30, -Tez vertex scope-20 -> Tez vertex scope-30, -Tez vertex scope-30 -> Tez vertex scope-32, -Tez vertex scope-32 +Tez vertex scope-11 -> Tez vertex scope-26, +Tez vertex scope-26 -> Tez vertex scope-36, +Tez vertex scope-36 -> Tez vertex scope-37, +Tez vertex scope-37 Tez vertex scope-11 # Plan on vertex -Local Rearrange[tuple]{tuple}(false) - scope-14 -> scope-20 +Local Rearrange[tuple]{tuple}(false) - scope-20 -> scope-26 | | -| Constant(DummyVal) - scope-13 +| Constant(DummyVal) - scope-19 | -|---New For Each(false,true)[tuple] - scope-19 +|---New For Each(false,true)[tuple] - scope-25 | | | Project[int][0] - scope-8 | | - | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-18 + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-24 | | - | |---Project[tuple][*] - scope-17 + | |---Project[tuple][*] - scope-23 | - |---ReservoirSample - scope-16 + |---ReservoirSample - scope-22 | - |---b: Local Rearrange[tuple]{int}(false) - scope-12 -> scope-30 + |---a: New For Each(false,false)[bag] - scope-7 | | - | Project[int][0] - scope-8 + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 | - |---a: New For Each(false,false)[bag] - scope-7 - | | - | Cast[int] - scope-2 - | | - | |---Project[bytearray][0] - scope-1 - | | - | Cast[int] - scope-5 - | | - | |---Project[bytearray][1] - scope-4 - | - |---a: Load(file:///tmp/input:PigStorage(',')) - scope-0 -Tez vertex scope-20 + |---a: Load(file:///tmp/input:PigStorage(',')) - scope-0 +Tez vertex scope-26 # Plan on vertex -POValueOutputTez - scope-29 -> [scope-30] +POValueOutputTez - scope-35 -> [scope-36] | -|---New For Each(false)[tuple] - scope-28 +|---New For Each(false)[tuple] - scope-34 | | - | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-27 + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-33 | | - | |---Project[tuple][*] - scope-26 + | |---Project[tuple][*] - scope-32 | - |---New For Each(false,false)[tuple] - scope-25 + |---New For Each(false,false)[tuple] - scope-31 | | - | Constant(-1) - scope-24 + | Constant(-1) - scope-30 | | - | Project[bag][1] - scope-22 + | Project[bag][1] - scope-28 | - |---Package(Packager)[tuple]{bytearray} - scope-21 -Tez vertex scope-30 + |---Package(Packager)[tuple]{bytearray} - scope-27 +Tez vertex scope-36 # Plan on vertex -POIdentityInOutTez - scope-31 <- scope-11 -> scope-32 +b: Local Rearrange[tuple]{int}(false) - scope-12 -> scope-37 | | | Project[int][0] - scope-8 -Tez vertex scope-32 +| +|---a: New For Each(false,false)[bag] - scope-18 + | | + | Cast[int] - scope-15 + | | + | |---Project[bytearray][0] - scope-14 + | | + | Cast[int] - scope-17 + | | + | |---Project[bytearray][1] - scope-16 + | + |---a: Load(file:///tmp/input:PigStorage(',')) - scope-13 +Tez vertex scope-37 # Plan on vertex b: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-10 | -|---New For Each(true)[tuple] - scope-35 +|---New For Each(true)[tuple] - scope-40 | | - | Project[bag][1] - scope-34 + | Project[bag][1] - scope-39 | - |---Package(LitePackager)[tuple]{int} - scope-33 + |---Package(LitePackager)[tuple]{int} - scope-38 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld Fri Mar 4 18:17:39 2016 @@ -4,68 +4,76 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-27 -> Tez vertex scope-36,Tez vertex scope-46, -Tez vertex scope-36 -> Tez vertex scope-28,Tez vertex scope-46, -Tez vertex scope-46 -> Tez vertex scope-50, -Tez vertex scope-28 -> Tez vertex scope-50, -Tez vertex scope-50 +Tez vertex scope-27 -> Tez vertex scope-42, +Tez vertex scope-42 -> Tez vertex scope-28,Tez vertex scope-52, +Tez vertex scope-52 -> Tez vertex scope-55, +Tez vertex scope-28 -> Tez vertex scope-55, +Tez vertex scope-55 Tez vertex scope-27 # Plan on vertex -Local Rearrange[tuple]{tuple}(false) - scope-31 -> scope-36 +Local Rearrange[tuple]{tuple}(false) - scope-31 -> scope-42 | | | Constant(DummyVal) - scope-30 | -|---New For Each(true,true)[tuple] - scope-35 +|---New For Each(true,true)[tuple] - scope-41 | | | Project[int][0] - scope-16 | | - | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-34 + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-40 | | - | |---Project[tuple][*] - scope-33 + | |---Project[tuple][*] - scope-39 | |---PoissonSample - scope-32 | - |---Local Rearrange[tuple]{int}(false) - scope-29 -> scope-46 + |---a: New For Each(false,false)[bag] - scope-7 | | - | Project[int][0] - scope-16 + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 | - |---a: New For Each(false,false)[bag] - scope-7 - | | - | Cast[int] - scope-2 - | | - | |---Project[bytearray][0] - scope-1 - | | - | Cast[int] - scope-5 - | | - | |---Project[bytearray][1] - scope-4 - | - |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-36 + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-42 # Plan on vertex -POValueOutputTez - scope-45 -> [scope-28, scope-46] +POValueOutputTez - scope-51 -> [scope-28, scope-52] | -|---New For Each(false)[tuple] - scope-44 +|---New For Each(false)[tuple] - scope-50 | | - | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-43 + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-49 | | - | |---Project[tuple][*] - scope-42 + | |---Project[tuple][*] - scope-48 | - |---New For Each(false,false)[tuple] - scope-41 + |---New For Each(false,false)[tuple] - scope-47 | | - | Constant(-1) - scope-40 + | Constant(-1) - scope-46 | | - | Project[bag][1] - scope-38 + | Project[bag][1] - scope-44 | - |---Package(Packager)[tuple]{bytearray} - scope-37 -Tez vertex scope-46 + |---Package(Packager)[tuple]{bytearray} - scope-43 +Tez vertex scope-52 # Plan on vertex -POIdentityInOutTez - scope-47 <- scope-27 -> scope-50 +Local Rearrange[tuple]{int}(false) - scope-29 -> scope-55 | | | Project[int][0] - scope-16 +| +|---a: New For Each(false,false)[bag] - scope-38 + | | + | Cast[int] - scope-35 + | | + | |---Project[bytearray][0] - scope-34 + | | + | Cast[int] - scope-37 + | | + | |---Project[bytearray][1] - scope-36 + | + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-33 Tez vertex scope-28 # Plan on vertex -Partition Rearrange[tuple]{int}(false) - scope-48 -> scope-50 +Partition Rearrange[tuple]{int}(false) - scope-53 -> scope-55 | | | Project[int][0] - scope-17 | @@ -80,7 +88,7 @@ Partition Rearrange[tuple]{int}(false) - | |---Project[bytearray][1] - scope-12 | |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8 -Tez vertex scope-50 +Tez vertex scope-55 # Plan on vertex d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 | @@ -92,10 +100,10 @@ d: Store(file:///tmp/output:org.apache.p | | | Project[int][3] - scope-23 | - |---New For Each(true,true)[tuple] - scope-54 + |---New For Each(true,true)[tuple] - scope-59 | | - | Project[bag][1] - scope-52 + | Project[bag][1] - scope-57 | | - | Project[bag][2] - scope-53 + | Project[bag][2] - scope-58 | - |---Package(Packager)[tuple]{int} - scope-51 + |---Package(Packager)[tuple]{int} - scope-56 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld Fri Mar 4 18:17:39 2016 @@ -2,44 +2,44 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: pig-0_scope-1 +# TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-45 -> Tez vertex scope-47, -Tez vertex scope-46 -> Tez vertex scope-47, -Tez vertex scope-47 +Tez vertex scope-18 -> Tez vertex scope-20, +Tez vertex scope-19 -> Tez vertex scope-20, +Tez vertex scope-20 -Tez vertex scope-45 +Tez vertex scope-18 # Plan on vertex -POValueOutputTez - scope-49 -> [scope-47] +POValueOutputTez - scope-22 -> [scope-20] | -|---a: New For Each(false,false)[bag] - scope-34 +|---a: New For Each(false,false)[bag] - scope-7 | | - | Cast[int] - scope-29 + | Cast[int] - scope-2 | | - | |---Project[bytearray][0] - scope-28 + | |---Project[bytearray][0] - scope-1 | | - | Cast[chararray] - scope-32 + | Cast[chararray] - scope-5 | | - | |---Project[bytearray][1] - scope-31 + | |---Project[bytearray][1] - scope-4 | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-27 -Tez vertex scope-46 + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-19 # Plan on vertex -POValueOutputTez - scope-50 -> [scope-47] +POValueOutputTez - scope-23 -> [scope-20] | -|---c: New For Each(false,false)[bag] - scope-42 +|---c: New For Each(false,false)[bag] - scope-15 | | - | Cast[int] - scope-37 + | Cast[int] - scope-10 | | - | |---Project[bytearray][1] - scope-36 + | |---Project[bytearray][1] - scope-9 | | - | Cast[chararray] - scope-40 + | Cast[chararray] - scope-13 | | - | |---Project[bytearray][0] - scope-39 + | |---Project[bytearray][0] - scope-12 | - |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-35 -Tez vertex scope-47 + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-20 # Plan on vertex -c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-44 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-17 | -|---POShuffledValueInputTez - scope-48 <- [scope-45, scope-46] +|---POShuffledValueInputTez - scope-21 <- [scope-18, scope-19] Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld Fri Mar 4 18:17:39 2016 @@ -10,7 +10,7 @@ Tez vertex group scope-24 Tez vertex scope-18 # Plan on vertex -c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25 -> scope-17 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -25,7 +25,7 @@ c: Store(file:///tmp/output:org.apache.p |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-19 # Plan on vertex -c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 -> scope-17 | |---c: New For Each(false,false)[bag] - scope-15 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld Fri Mar 4 18:17:39 2016 @@ -4,16 +4,16 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-31 -> Tez vertex scope-33, -Tez vertex scope-32 -> Tez vertex scope-33, -Tez vertex scope-33 -> Tez vertex scope-38, -Tez vertex scope-37 -> Tez vertex scope-38, -Tez vertex scope-38 -> Tez vertex scope-42, -Tez vertex scope-42 +Tez vertex scope-37 -> Tez vertex scope-39, +Tez vertex scope-38 -> Tez vertex scope-39, +Tez vertex scope-39 -> Tez vertex scope-44, +Tez vertex scope-43 -> Tez vertex scope-44, +Tez vertex scope-44 -> Tez vertex scope-53, +Tez vertex scope-53 -Tez vertex scope-31 +Tez vertex scope-37 # Plan on vertex -POValueOutputTez - scope-35 -> [scope-33] +POValueOutputTez - scope-41 -> [scope-39] | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -26,9 +26,9 @@ POValueOutputTez - scope-35 -> [scope-3 | |---Project[bytearray][1] - scope-4 | |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-32 +Tez vertex scope-38 # Plan on vertex -POValueOutputTez - scope-36 -> [scope-33] +POValueOutputTez - scope-42 -> [scope-39] | |---c: New For Each(false,false)[bag] - scope-15 | | @@ -41,14 +41,14 @@ POValueOutputTez - scope-36 -> [scope-3 | |---Project[bytearray][0] - scope-12 | |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 -Tez vertex scope-33 +Tez vertex scope-39 # Plan on vertex -POValueOutputTez - scope-40 -> [scope-38] +POValueOutputTez - scope-46 -> [scope-44] | -|---POShuffledValueInputTez - scope-34 <- [scope-31, scope-32] -Tez vertex scope-37 +|---POShuffledValueInputTez - scope-40 <- [scope-37, scope-38] +Tez vertex scope-43 # Plan on vertex -POValueOutputTez - scope-41 -> [scope-38] +POValueOutputTez - scope-47 -> [scope-44] | |---d: New For Each(false,false)[bag] - scope-24 | | @@ -61,15 +61,19 @@ POValueOutputTez - scope-41 -> [scope-3 | |---Project[bytearray][1] - scope-21 | |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17 -Tez vertex scope-38 +Tez vertex scope-44 # Plan on vertex -f: Local Rearrange[tuple]{int}(false) - scope-28 -> scope-42 +e: Split - scope-54 +| | +| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-29 | | -| Project[int][0] - scope-29 +| f: Local Rearrange[tuple]{int}(false) - scope-34 -> scope-53 +| | | +| | Project[int][0] - scope-35 | -|---POShuffledValueInputTez - scope-39 <- [scope-33, scope-37] -Tez vertex scope-42 +|---POShuffledValueInputTez - scope-45 <- [scope-39, scope-43] +Tez vertex scope-53 # Plan on vertex -f: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-30 +f: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-36 | -|---f: Package(Packager)[tuple]{int} - scope-27 +|---f: Package(Packager)[tuple]{int} - scope-33 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld Fri Mar 4 18:17:39 2016 @@ -4,17 +4,22 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-37 -> Tez vertex group scope-43, -Tez vertex scope-31 -> Tez vertex group scope-43, -Tez vertex scope-32 -> Tez vertex group scope-43, -Tez vertex group scope-43 -> Tez vertex scope-42, -Tez vertex scope-42 +Tez vertex scope-43 -> Tez vertex group scope-55,Tez vertex group scope-56, +Tez vertex scope-37 -> Tez vertex group scope-55,Tez vertex group scope-56, +Tez vertex scope-38 -> Tez vertex group scope-55,Tez vertex group scope-56, +Tez vertex group scope-56 -> Tez vertex scope-53, +Tez vertex scope-53 +Tez vertex group scope-55 -Tez vertex scope-37 +Tez vertex scope-43 # Plan on vertex -f: Local Rearrange[tuple]{int}(false) - scope-46 -> scope-42 +e: Split - scope-61 +| | +| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-62 -> scope-29 | | -| Project[int][0] - scope-47 +| f: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-53 +| | | +| | Project[int][0] - scope-64 | |---d: New For Each(false,false)[bag] - scope-24 | | @@ -27,11 +32,15 @@ f: Local Rearrange[tuple]{int}(false) - | |---Project[bytearray][1] - scope-21 | |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17 -Tez vertex scope-31 +Tez vertex scope-37 # Plan on vertex -f: Local Rearrange[tuple]{int}(false) - scope-49 -> scope-42 +e: Split - scope-66 | | -| Project[int][0] - scope-50 +| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-67 -> scope-29 +| | +| f: Local Rearrange[tuple]{int}(false) - scope-68 -> scope-53 +| | | +| | Project[int][0] - scope-69 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -44,11 +53,15 @@ f: Local Rearrange[tuple]{int}(false) - | |---Project[bytearray][1] - scope-4 | |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-32 +Tez vertex scope-38 # Plan on vertex -f: Local Rearrange[tuple]{int}(false) - scope-51 -> scope-42 +e: Split - scope-70 +| | +| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-71 -> scope-29 | | -| Project[int][0] - scope-52 +| f: Local Rearrange[tuple]{int}(false) - scope-72 -> scope-53 +| | | +| | Project[int][0] - scope-73 | |---c: New For Each(false,false)[bag] - scope-15 | | @@ -61,10 +74,12 @@ f: Local Rearrange[tuple]{int}(false) - | |---Project[bytearray][0] - scope-12 | |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 -Tez vertex group scope-43 <- [scope-37, scope-31, scope-32] -> scope-42 +Tez vertex group scope-56 <- [scope-43, scope-37, scope-38] -> scope-53 # No plan on vertex group -Tez vertex scope-42 +Tez vertex scope-53 # Plan on vertex -f: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-30 +f: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-36 | -|---f: Package(Packager)[tuple]{int} - scope-27 +|---f: Package(Packager)[tuple]{int} - scope-33 +Tez vertex group scope-55 <- [scope-43, scope-37, scope-38] -> null +# No plan on vertex group Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld Fri Mar 4 18:17:39 2016 @@ -4,84 +4,84 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-1 #-------------------------------------------------- -Tez vertex scope-101 -> Tez vertex scope-103, -Tez vertex scope-102 -> Tez vertex scope-103, -Tez vertex scope-103 -> Tez vertex scope-107, -Tez vertex scope-107 +Tez vertex scope-103 -> Tez vertex scope-105, +Tez vertex scope-104 -> Tez vertex scope-105, +Tez vertex scope-105 -> Tez vertex scope-109, +Tez vertex scope-109 -Tez vertex scope-101 +Tez vertex scope-103 # Plan on vertex -POValueOutputTez - scope-105 -> [scope-103] +POValueOutputTez - scope-107 -> [scope-105] | -|---a: New For Each(false,false)[bag] - scope-79 +|---a: New For Each(false,false)[bag] - scope-81 | | - | Cast[int] - scope-74 + | Cast[int] - scope-76 | | - | |---Project[bytearray][0] - scope-73 + | |---Project[bytearray][0] - scope-75 | | - | Cast[int] - scope-77 + | Cast[int] - scope-79 | | - | |---Project[bytearray][1] - scope-76 + | |---Project[bytearray][1] - scope-78 | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-72 -Tez vertex scope-102 + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-74 +Tez vertex scope-104 # Plan on vertex -POValueOutputTez - scope-106 -> [scope-103] +POValueOutputTez - scope-108 -> [scope-105] | -|---c: New For Each(false,false)[bag] - scope-87 +|---c: New For Each(false,false)[bag] - scope-89 | | - | Cast[int] - scope-82 + | Cast[int] - scope-84 | | - | |---Project[bytearray][1] - scope-81 + | |---Project[bytearray][1] - scope-83 | | - | Cast[int] - scope-85 + | Cast[int] - scope-87 | | - | |---Project[bytearray][0] - scope-84 + | |---Project[bytearray][0] - scope-86 | - |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-80 -Tez vertex scope-103 + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-82 +Tez vertex scope-105 # Plan on vertex -d: Local Rearrange[tuple]{int}(false) - scope-120 -> scope-107 +d: Local Rearrange[tuple]{int}(false) - scope-122 -> scope-109 | | -| Project[int][0] - scope-122 +| Project[int][0] - scope-124 | -|---e: New For Each(false,false)[bag] - scope-108 +|---e: New For Each(false,false)[bag] - scope-110 | | - | Project[int][0] - scope-109 + | Project[int][0] - scope-111 | | - | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-110 + | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-112 | | - | |---Project[bag][1] - scope-111 + | |---Project[bag][1] - scope-113 | | - | |---Project[bag][1] - scope-112 + | |---Project[bag][1] - scope-114 | - |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-123 + |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-125 | - |---POShuffledValueInputTez - scope-104 <- [scope-101, scope-102] -Tez vertex scope-107 -# Combine plan on edge <scope-103> -d: Local Rearrange[tuple]{int}(false) - scope-124 -> scope-107 + |---POShuffledValueInputTez - scope-106 <- [scope-103, scope-104] +Tez vertex scope-109 +# Combine plan on edge <scope-105> +d: Local Rearrange[tuple]{int}(false) - scope-126 -> scope-109 | | -| Project[int][0] - scope-126 +| Project[int][0] - scope-128 | -|---e: New For Each(false,false)[bag] - scope-113 +|---e: New For Each(false,false)[bag] - scope-115 | | - | Project[int][0] - scope-114 + | Project[int][0] - scope-116 | | - | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-115 + | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-117 | | - | |---Project[bag][1] - scope-116 + | |---Project[bag][1] - scope-118 | - |---d: Package(CombinerPackager)[tuple]{int} - scope-119 + |---d: Package(CombinerPackager)[tuple]{int} - scope-121 # Plan on vertex -e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-100 +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-102 | -|---e: New For Each(false,false)[bag] - scope-99 +|---e: New For Each(false,false)[bag] - scope-101 | | - | Project[int][0] - scope-93 + | Project[int][0] - scope-95 | | - | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-97 + | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-99 | | - | |---Project[bag][1] - scope-117 + | |---Project[bag][1] - scope-119 | - |---d: Package(CombinerPackager)[tuple]{int} - scope-90 + |---d: Package(CombinerPackager)[tuple]{int} - scope-92 Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld Fri Mar 4 18:17:39 2016 @@ -40,21 +40,21 @@ d: Local Rearrange[tuple]{int}(false) - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-30 # Plan on vertex -d: Local Rearrange[tuple]{int}(false) - scope-69 -> scope-35 +d: Local Rearrange[tuple]{int}(false) - scope-70 -> scope-35 | | -| Project[int][0] - scope-70 +| Project[int][0] - scope-71 | -|---e: New For Each(false,false)[bag] - scope-68 +|---e: New For Each(false,false)[bag] - scope-69 | | - | Project[int][0] - scope-64 + | Project[int][0] - scope-65 | | - | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-65 + | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-66 | | - | |---Project[bag][1] - scope-66 + | |---Project[bag][1] - scope-67 | | - | |---Project[bag][1] - scope-67 + | |---Project[bag][1] - scope-68 | - |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-71 + |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-72 | |---c: New For Each(false,false)[bag] - scope-15 | | Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld Fri Mar 4 18:17:39 2016 @@ -2,88 +2,98 @@ # There are 1 DAGs in the session #-------------------------------------------------- #-------------------------------------------------- -# TEZ DAG plan: pig-0_scope-1 +# TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-108 -> Tez vertex scope-109, -Tez vertex scope-102 -> Tez vertex scope-109, -Tez vertex scope-109 +Tez vertex scope-45 -> Tez vertex scope-52,Tez vertex scope-57, +Tez vertex scope-52 -> Tez vertex scope-57, +Tez vertex scope-56 -> Tez vertex scope-57, +Tez vertex scope-57 -Tez vertex scope-108 +Tez vertex scope-45 # Plan on vertex -POValueOutputTez - scope-112 -> [scope-109] -| -|---c: New For Each(false,false)[bag] - scope-88 - | | - | Cast[int] - scope-83 - | | - | |---Project[bytearray][1] - scope-82 - | | - | Cast[chararray] - scope-86 - | | - | |---Project[bytearray][0] - scope-85 - | - |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-81 -Tez vertex scope-102 -# Plan on vertex -1-12: Split - scope-119 +1-2: Split - scope-69 | | -| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-76 +| a2: Split - scope-68 +| | | +| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17 +| | | +| | POValueOutputTez - scope-60 -> [scope-57] | | -| |---a2: Filter[bag] - scope-71 +| |---a2: Filter[bag] - scope-9 | | | -| | Not[boolean] - scope-75 +| | Not[boolean] - scope-13 | | | -| | |---Greater Than[boolean] - scope-74 +| | |---Greater Than[boolean] - scope-12 | | | -| | |---Project[int][0] - scope-72 +| | |---Project[int][0] - scope-10 | | | -| | |---Constant(100) - scope-73 -| | -| POValueOutputTez - scope-111 -> [scope-109] +| | |---Constant(100) - scope-11 | | -| |---a1: Filter[bag] - scope-77 -| | | -| | Greater Than[boolean] - scope-80 -| | | -| | |---Project[int][0] - scope-78 -| | | -| | |---Constant(100) - scope-79 +| POValueOutputTez - scope-46 -> [scope-52] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-52 +# Plan on vertex +POValueOutputTez - scope-59 -> [scope-57] +| +|---a1: Filter[bag] - scope-18 + | | + | Greater Than[boolean] - scope-21 + | | + | |---Project[int][0] - scope-19 + | | + | |---Constant(100) - scope-20 + | + |---POValueInputTez - scope-53 <- scope-45 +Tez vertex scope-56 +# Plan on vertex +POValueOutputTez - scope-61 -> [scope-57] | -|---a: New For Each(false,false)[bag] - scope-69 +|---c: New For Each(false,false)[bag] - scope-31 | | - | Cast[int] - scope-64 + | Cast[int] - scope-26 | | - | |---Project[bytearray][0] - scope-63 + | |---Project[bytearray][1] - scope-25 | | - | Cast[chararray] - scope-67 + | Cast[chararray] - scope-29 | | - | |---Project[bytearray][1] - scope-66 + | |---Project[bytearray][0] - scope-28 | - |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-62 -Tez vertex scope-109 + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24 +Tez vertex scope-57 # Plan on vertex -1-13: Split - scope-118 +1-3: Split - scope-67 | | -| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-95 +| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-38 | | -| |---d: Filter[bag] - scope-91 +| |---d: Filter[bag] - scope-34 | | | -| | Greater Than[boolean] - scope-94 +| | Greater Than[boolean] - scope-37 | | | -| | |---Project[int][0] - scope-92 +| | |---Project[int][0] - scope-35 | | | -| | |---Constant(500) - scope-93 +| | |---Constant(500) - scope-36 | | -| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-101 +| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-44 | | -| |---e: Filter[bag] - scope-96 +| |---e: Filter[bag] - scope-39 | | | -| | Not[boolean] - scope-100 +| | Not[boolean] - scope-43 | | | -| | |---Greater Than[boolean] - scope-99 +| | |---Greater Than[boolean] - scope-42 | | | -| | |---Project[int][0] - scope-97 +| | |---Project[int][0] - scope-40 | | | -| | |---Constant(500) - scope-98 +| | |---Constant(500) - scope-41 | -|---POShuffledValueInputTez - scope-110 <- [scope-102, scope-108] +|---POShuffledValueInputTez - scope-58 <- [scope-45, scope-52, scope-56] Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld (original) +++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld Fri Mar 4 18:17:39 2016 @@ -4,16 +4,42 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-40 -> Tez vertex group scope-58,Tez vertex group scope-59, -Tez vertex scope-46 -> Tez vertex group scope-58,Tez vertex group scope-59, -Tez vertex group scope-58 -Tez vertex group scope-59 +Tez vertex scope-45 -> Tez vertex group scope-70,Tez vertex group scope-70,Tez vertex group scope-71,Tez vertex group scope-71, +Tez vertex scope-56 -> Tez vertex group scope-70,Tez vertex group scope-71, +Tez vertex group scope-70 +Tez vertex group scope-71 -Tez vertex scope-40 +Tez vertex scope-45 # Plan on vertex -1-2: Split - scope-57 +1-2: Split - scope-69 | | -| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14 +| a2: Split - scope-68 +| | | +| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17 +| | | +| | 1-3: Split - scope-72 +| | | | +| | | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-77 -> scope-38 +| | | | +| | | |---d: Filter[bag] - scope-73 +| | | | | +| | | | Greater Than[boolean] - scope-76 +| | | | | +| | | | |---Project[int][0] - scope-74 +| | | | | +| | | | |---Constant(500) - scope-75 +| | | | +| | | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-83 -> scope-44 +| | | | +| | | |---e: Filter[bag] - scope-78 +| | | | | +| | | | Not[boolean] - scope-82 +| | | | | +| | | | |---Greater Than[boolean] - scope-81 +| | | | | +| | | | |---Project[int][0] - scope-79 +| | | | | +| | | | |---Constant(500) - scope-80 | | | |---a2: Filter[bag] - scope-9 | | | @@ -25,37 +51,37 @@ Tez vertex scope-40 | | | | | |---Constant(100) - scope-11 | | -| 1-3: Split - scope-60 +| 1-3: Split - scope-84 | | | -| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33 +| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-89 -> scope-38 | | | -| | |---d: Filter[bag] - scope-29 +| | |---d: Filter[bag] - scope-85 | | | | -| | | Greater Than[boolean] - scope-32 +| | | Greater Than[boolean] - scope-88 | | | | -| | | |---Project[int][0] - scope-30 +| | | |---Project[int][0] - scope-86 | | | | -| | | |---Constant(500) - scope-31 +| | | |---Constant(500) - scope-87 | | | -| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39 +| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-95 -> scope-44 | | | -| | |---e: Filter[bag] - scope-34 +| | |---e: Filter[bag] - scope-90 | | | | -| | | Not[boolean] - scope-38 +| | | Not[boolean] - scope-94 | | | | -| | | |---Greater Than[boolean] - scope-37 +| | | |---Greater Than[boolean] - scope-93 | | | | -| | | |---Project[int][0] - scope-35 +| | | |---Project[int][0] - scope-91 | | | | -| | | |---Constant(500) - scope-36 +| | | |---Constant(500) - scope-92 | | -| |---a1: Filter[bag] - scope-15 +| |---a1: Filter[bag] - scope-18 | | | -| | Greater Than[boolean] - scope-18 +| | Greater Than[boolean] - scope-21 | | | -| | |---Project[int][0] - scope-16 +| | |---Project[int][0] - scope-19 | | | -| | |---Constant(100) - scope-17 +| | |---Constant(100) - scope-20 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -68,44 +94,44 @@ Tez vertex scope-40 | |---Project[bytearray][1] - scope-4 | |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-46 +Tez vertex scope-56 # Plan on vertex -1-3: Split - scope-61 +1-3: Split - scope-96 | | -| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33 +| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-101 -> scope-38 | | -| |---d: Filter[bag] - scope-29 +| |---d: Filter[bag] - scope-97 | | | -| | Greater Than[boolean] - scope-32 +| | Greater Than[boolean] - scope-100 | | | -| | |---Project[int][0] - scope-30 +| | |---Project[int][0] - scope-98 | | | -| | |---Constant(500) - scope-31 +| | |---Constant(500) - scope-99 | | -| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39 +| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-107 -> scope-44 | | -| |---e: Filter[bag] - scope-34 +| |---e: Filter[bag] - scope-102 | | | -| | Not[boolean] - scope-38 +| | Not[boolean] - scope-106 | | | -| | |---Greater Than[boolean] - scope-37 +| | |---Greater Than[boolean] - scope-105 | | | -| | |---Project[int][0] - scope-35 +| | |---Project[int][0] - scope-103 | | | -| | |---Constant(500) - scope-36 +| | |---Constant(500) - scope-104 | -|---c: New For Each(false,false)[bag] - scope-26 +|---c: New For Each(false,false)[bag] - scope-31 | | - | Cast[int] - scope-21 + | Cast[int] - scope-26 | | - | |---Project[bytearray][1] - scope-20 + | |---Project[bytearray][1] - scope-25 | | - | Cast[chararray] - scope-24 + | Cast[chararray] - scope-29 | | - | |---Project[bytearray][0] - scope-23 + | |---Project[bytearray][0] - scope-28 | - |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19 -Tez vertex group scope-58 <- [scope-40, scope-46] -> null + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24 +Tez vertex group scope-70 <- [scope-45, scope-45, scope-56] -> null # No plan on vertex group -Tez vertex group scope-59 <- [scope-40, scope-46] -> null +Tez vertex group scope-71 <- [scope-45, scope-45, scope-56] -> null # No plan on vertex group Modified: pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java (original) +++ pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java Fri Mar 4 18:17:39 2016 @@ -26,6 +26,7 @@ import java.io.PrintWriter; import java.lang.String; import java.util.HashMap; import java.util.Map; +import java.util.Properties; import junit.framework.ComparisonFailure; @@ -38,6 +39,7 @@ import org.apache.pig.pigunit.PigTest; import org.apache.pig.pigunit.pig.PigServer; import org.apache.pig.test.Util; import org.apache.pig.tools.parameters.ParseException; +import org.apache.pig.impl.util.PropertiesUtil; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -378,16 +380,13 @@ public class TestPigTest { @Test public void testDefaultBootup() throws Exception { - // Test with properties file - String pigProps = "pig.properties"; + // Test with properties object + Properties pigProps = PropertiesUtil.loadDefaultProperties(); String bootupPath = "/tmp/.temppigbootup"; - File propertyFile = new File(pigProps); - PrintWriter out = new PrintWriter(new FileWriter(propertyFile)); - out.println("pig.load.default.statements=" + bootupPath); - out.close(); + pigProps.setProperty("pig.load.default.statements", bootupPath); File bootupFile = new File(bootupPath); - out = new PrintWriter(new FileWriter(bootupFile)); + PrintWriter out = new PrintWriter(new FileWriter(bootupFile)); out.println("data = LOAD 'top_queries_input_data.txt' AS (query:CHARARRAY, count:INT);"); out.close(); @@ -420,7 +419,7 @@ public class TestPigTest { // Create a pigunit.pig.PigServer and Cluster to run this test. PigServer pig = null; - pig = new PigServer(Util.getLocalTestMode()); + pig = new PigServer(Util.getLocalTestMode(), pigProps); final Cluster cluster = new Cluster(pig.getPigContext()); @@ -434,7 +433,6 @@ public class TestPigTest { test.assertOutput("queries_limit", output); - propertyFile.delete(); scriptFile.delete(); bootupFile.delete(); } Modified: pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java (original) +++ pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java Fri Mar 4 18:17:39 2016 @@ -20,6 +20,9 @@ package org.apache.pig.tez; import static org.junit.Assert.assertEquals; +import java.util.HashMap; + +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder; import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler; @@ -68,7 +71,7 @@ public class TestGroupConstParallelTez e parallelismSetter.visit(); DAG tezDag = getTezDAG(tezPlan, pc); - TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null); + TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>()); dagBuilder.visit(); for (Vertex v : tezDag.getVertices()) { if (!v.getInputVertices().isEmpty()) { @@ -88,7 +91,7 @@ public class TestGroupConstParallelTez e parallelismSetter.visit(); DAG tezDag = getTezDAG(tezPlan, pc); - TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null); + TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>()); dagBuilder.visit(); for (Vertex v : tezDag.getVertices()) { if (!v.getInputVertices().isEmpty()) { Modified: pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java (original) +++ pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java Fri Mar 4 18:17:39 2016 @@ -20,7 +20,10 @@ package org.apache.pig.tez; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.HashMap; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.pig.backend.hadoop.executionengine.JobCreationException; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder; @@ -62,7 +65,7 @@ public class TestJobSubmissionTez extend parallelismSetter.visit(); DAG tezDag = getTezDAG(tezPlan, pc); - TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null); + TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>()); try { dagBuilder.visit(); } catch (VisitorException jce) { @@ -81,7 +84,7 @@ public class TestJobSubmissionTez extend parallelismSetter.visit(); DAG tezDag = getTezDAG(tezPlan, pc); - TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null); + TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>()); dagBuilder.visit(); for (Vertex v : tezDag.getVertices()) { if (!v.getInputVertices().isEmpty()) { Modified: pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java?rev=1733627&r1=1733626&r2=1733627&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java (original) +++ pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java Fri Mar 4 18:17:39 2016 @@ -20,6 +20,7 @@ package org.apache.pig.tez; import static org.junit.Assert.assertEquals; import java.io.File; +import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.pig.backend.executionengine.ExecJob; @@ -38,7 +39,7 @@ import org.apache.pig.tools.pigstats.tez public class TestPigStatsTez extends TestPigStats { @Override - public void addSettingsToConf(Configuration conf, String scriptFileName) { + public void addSettingsToConf(Configuration conf, String scriptFileName) throws IOException { TezScriptState ss = TezScriptState.get(); ss.setScript(new File(scriptFileName)); ss.addDAGSettingsToConf(conf);
