Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,121 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-30 -> Tez vertex scope-48,Tez vertex scope-58, +Tez vertex scope-48 -> Tez vertex scope-40,Tez vertex scope-58, +Tez vertex scope-58 -> Tez vertex scope-62, +Tez vertex scope-40 -> Tez vertex scope-62, +Tez vertex scope-62 + +Tez vertex scope-30 +# Plan on vertex +a: Split - scope-68 +| | +| Local Rearrange[tuple]{tuple}(false) - scope-71 -> scope-48 +| | | +| | Constant(DummyVal) - scope-72 +| | +| |---New For Each(true,true)[tuple] - scope-78 +| | | +| | Project[int][0] - scope-75 +| | | +| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-77 +| | | +| | |---Project[tuple][*] - scope-76 +| | +| |---PoissonSample - scope-74 +| | +| |---Local Rearrange[tuple]{int}(false) - scope-69 -> scope-58 +| | | +| | Project[int][0] - scope-70 +| | +| Local Rearrange[tuple]{tuple}(false) - scope-81 -> scope-48 +| | | +| | Constant(DummyVal) - scope-82 +| | +| |---New For Each(true,true)[tuple] - scope-88 +| | | +| | Project[int][0] - scope-85 +| | | +| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-87 +| | | +| | |---Project[tuple][*] - scope-86 +| | +| |---PoissonSample - scope-84 +| | +| |---Local Rearrange[tuple]{int}(false) - scope-79 -> scope-58 +| | | +| | Project[int][0] - scope-80 +| | +| |---b: Filter[bag] - scope-13 +| | | +| | Equal To[boolean] - scope-16 +| | | +| | |---Project[int][0] - scope-14 +| | | +| | |---Constant(2) - scope-15 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-48 +# Plan on vertex +POValueOutputTez - scope-57 -> [scope-40, scope-58] +| +|---New For Each(false)[tuple] - scope-56 + | | + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55 + | | + | |---Project[tuple][*] - scope-54 + | + |---New For Each(false,false)[tuple] - scope-53 + | | + | Constant(-1) - scope-52 + | | + | Project[bag][1] - scope-50 + | + |---Package(Packager)[tuple]{bytearray} - scope-49 +Tez vertex scope-58 +# Plan on vertex +POIdentityInOutTez - scope-59 <- scope-30 -> scope-62 +| | +| Project[int][0] - scope-26 +Tez vertex scope-40 +# Plan on vertex +Partition Rearrange[tuple]{int}(false) - scope-60 -> scope-62 +| | +| Project[int][0] - scope-27 +| +|---d: New For Each(false,false)[bag] - scope-25 + | | + | Cast[int] - scope-20 + | | + | |---Project[bytearray][0] - scope-19 + | | + | Cast[chararray] - scope-23 + | | + | |---Project[bytearray][1] - scope-22 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-18 +Tez vertex scope-62 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29 +| +|---New For Each(true,true)[tuple] - scope-66 + | | + | Project[bag][1] - scope-64 + | | + | Project[bag][2] - scope-65 + | + |---Package(Packager)[tuple]{int} - scope-63
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,117 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-30 -> Tez vertex scope-48,Tez vertex scope-58, +Tez vertex scope-48 -> Tez vertex scope-37,Tez vertex scope-58, +Tez vertex scope-58 -> Tez vertex scope-62, +Tez vertex scope-31 -> Tez vertex scope-35,Tez vertex scope-37, +Tez vertex scope-35 -> Tez vertex scope-37, +Tez vertex scope-37 -> Tez vertex scope-62, +Tez vertex scope-62 + +Tez vertex scope-30 +# Plan on vertex +Local Rearrange[tuple]{tuple}(false) - scope-43 -> scope-48 +| | +| Constant(DummyVal) - scope-42 +| +|---New For Each(true,true)[tuple] - scope-47 + | | + | Project[int][0] - scope-26 + | | + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-46 + | | + | |---Project[tuple][*] - scope-45 + | + |---PoissonSample - scope-44 + | + |---Local Rearrange[tuple]{int}(false) - scope-41 -> scope-58 + | | + | Project[int][0] - scope-26 + | + |---d: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-48 +# Plan on vertex +POValueOutputTez - scope-57 -> [scope-37, scope-58] +| +|---New For Each(false)[tuple] - scope-56 + | | + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55 + | | + | |---Project[tuple][*] - scope-54 + | + |---New For Each(false,false)[tuple] - scope-53 + | | + | Constant(-1) - scope-52 + | | + | Project[bag][1] - scope-50 + | + |---Package(Packager)[tuple]{bytearray} - scope-49 +Tez vertex scope-58 +# Plan on vertex +POIdentityInOutTez - scope-59 <- scope-30 -> scope-62 +| | +| Project[int][0] - scope-26 +Tez vertex scope-31 +# Plan on vertex +a: Split - scope-68 +| | +| POValueOutputTez - scope-39 -> [scope-37] +| | +| POValueOutputTez - scope-32 -> [scope-35] +| +|---a: New For Each(false,false)[bag] - scope-15 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][0] - scope-9 + | | + | Cast[chararray] - scope-13 + | | + | |---Project[bytearray][1] - scope-12 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-35 +# Plan on vertex +POValueOutputTez - scope-40 -> [scope-37] +| +|---b: Filter[bag] - scope-21 + | | + | Equal To[boolean] - scope-24 + | | + | |---Project[int][0] - scope-22 + | | + | |---Constant(2) - scope-23 + | + |---POValueInputTez - scope-36 <- scope-31 +Tez vertex scope-37 +# Plan on vertex +Partition Rearrange[tuple]{int}(false) - scope-60 -> scope-62 +| | +| Project[int][0] - scope-27 +| +|---POShuffledValueInputTez - scope-38 <- [scope-31, scope-35] +Tez vertex scope-62 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29 +| +|---New For Each(true,true)[tuple] - scope-66 + | | + | Project[bag][1] - scope-64 + | | + | Project[bag][2] - scope-65 + | + |---Package(Packager)[tuple]{int} - scope-63 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,107 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-30 -> Tez vertex scope-48,Tez vertex scope-58, +Tez vertex scope-48 -> Tez vertex scope-31,Tez vertex scope-58, +Tez vertex scope-58 -> Tez vertex scope-62, +Tez vertex scope-31 -> Tez vertex scope-62, +Tez vertex scope-62 + +Tez vertex scope-30 +# Plan on vertex +Local Rearrange[tuple]{tuple}(false) - scope-43 -> scope-48 +| | +| Constant(DummyVal) - scope-42 +| +|---New For Each(true,true)[tuple] - scope-47 + | | + | Project[int][0] - scope-26 + | | + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-46 + | | + | |---Project[tuple][*] - scope-45 + | + |---PoissonSample - scope-44 + | + |---Local Rearrange[tuple]{int}(false) - scope-41 -> scope-58 + | | + | Project[int][0] - scope-26 + | + |---d: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-48 +# Plan on vertex +POValueOutputTez - scope-57 -> [scope-31, scope-58] +| +|---New For Each(false)[tuple] - scope-56 + | | + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55 + | | + | |---Project[tuple][*] - scope-54 + | + |---New For Each(false,false)[tuple] - scope-53 + | | + | Constant(-1) - scope-52 + | | + | Project[bag][1] - scope-50 + | + |---Package(Packager)[tuple]{bytearray} - scope-49 +Tez vertex scope-58 +# Plan on vertex +POIdentityInOutTez - scope-59 <- scope-30 -> scope-62 +| | +| Project[int][0] - scope-26 +Tez vertex scope-31 +# Plan on vertex +a: Split - scope-68 +| | +| Partition Rearrange[tuple]{int}(false) - scope-69 -> scope-62 +| | | +| | Project[int][0] - scope-70 +| | +| Partition Rearrange[tuple]{int}(false) - scope-71 -> scope-62 +| | | +| | Project[int][0] - scope-72 +| | +| |---b: Filter[bag] - scope-21 +| | | +| | Equal To[boolean] - scope-24 +| | | +| | |---Project[int][0] - scope-22 +| | | +| | |---Constant(2) - scope-23 +| +|---a: New For Each(false,false)[bag] - scope-15 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][0] - scope-9 + | | + | Cast[chararray] - scope-13 + | | + | |---Project[bytearray][1] - scope-12 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-62 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29 +| +|---New For Each(true,true)[tuple] - scope-66 + | | + | Project[bag][1] - scope-64 + | | + | Project[bag][2] - scope-65 + | + |---Package(Packager)[tuple]{int} - scope-63 Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld (original) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -4,30 +4,20 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-46 -> Tez vertex scope-47, -Tez vertex scope-40 -> Tez vertex scope-47, -Tez vertex scope-47 +Tez vertex scope-45 -> Tez vertex scope-52,Tez vertex scope-57, +Tez vertex scope-52 -> Tez vertex scope-57, +Tez vertex scope-56 -> Tez vertex scope-57, +Tez vertex scope-57 -Tez vertex scope-46 +Tez vertex scope-45 # Plan on vertex -POValueOutputTez - scope-50 -> [scope-47] -| -|---c: New For Each(false,false)[bag] - scope-26 - | | - | Cast[int] - scope-21 - | | - | |---Project[bytearray][1] - scope-20 - | | - | Cast[chararray] - scope-24 - | | - | |---Project[bytearray][0] - scope-23 - | - |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19 -Tez vertex scope-40 -# Plan on vertex -1-2: Split - scope-57 +1-2: Split - scope-69 | | -| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14 +| a2: Split - scope-68 +| | | +| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17 +| | | +| | POValueOutputTez - scope-60 -> [scope-57] | | | |---a2: Filter[bag] - scope-9 | | | @@ -39,15 +29,7 @@ Tez vertex scope-40 | | | | | |---Constant(100) - scope-11 | | -| POValueOutputTez - scope-49 -> [scope-47] -| | -| |---a1: Filter[bag] - scope-15 -| | | -| | Greater Than[boolean] - scope-18 -| | | -| | |---Project[int][0] - scope-16 -| | | -| | |---Constant(100) - scope-17 +| POValueOutputTez - scope-46 -> [scope-52] | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -60,30 +42,58 @@ Tez vertex scope-40 | |---Project[bytearray][1] - scope-4 | |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-47 +Tez vertex scope-52 +# Plan on vertex +POValueOutputTez - scope-59 -> [scope-57] +| +|---a1: Filter[bag] - scope-18 + | | + | Greater Than[boolean] - scope-21 + | | + | |---Project[int][0] - scope-19 + | | + | |---Constant(100) - scope-20 + | + |---POValueInputTez - scope-53 <- scope-45 +Tez vertex scope-56 +# Plan on vertex +POValueOutputTez - scope-61 -> [scope-57] +| +|---c: New For Each(false,false)[bag] - scope-31 + | | + | Cast[int] - scope-26 + | | + | |---Project[bytearray][1] - scope-25 + | | + | Cast[chararray] - scope-29 + | | + | |---Project[bytearray][0] - scope-28 + | + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24 +Tez vertex scope-57 # Plan on vertex -1-3: Split - scope-56 +1-3: Split - scope-67 | | -| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33 +| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-38 | | -| |---d: Filter[bag] - scope-29 +| |---d: Filter[bag] - scope-34 | | | -| | Greater Than[boolean] - scope-32 +| | Greater Than[boolean] - scope-37 | | | -| | |---Project[int][0] - scope-30 +| | |---Project[int][0] - scope-35 | | | -| | |---Constant(500) - scope-31 +| | |---Constant(500) - scope-36 | | -| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39 +| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-44 | | -| |---e: Filter[bag] - scope-34 +| |---e: Filter[bag] - scope-39 | | | -| | Not[boolean] - scope-38 +| | Not[boolean] - scope-43 | | | -| | |---Greater Than[boolean] - scope-37 +| | |---Greater Than[boolean] - scope-42 | | | -| | |---Project[int][0] - scope-35 +| | |---Project[int][0] - scope-40 | | | -| | |---Constant(500) - scope-36 +| | |---Constant(500) - scope-41 | -|---POShuffledValueInputTez - scope-48 <- [scope-40, scope-46] +|---POShuffledValueInputTez - scope-58 <- [scope-45, scope-52, scope-56] Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld (original) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld Tue Apr 7 23:21:01 2015 @@ -4,16 +4,42 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-40 -> Tez vertex group scope-58,Tez vertex group scope-59, -Tez vertex scope-46 -> Tez vertex group scope-58,Tez vertex group scope-59, -Tez vertex group scope-58 -Tez vertex group scope-59 +Tez vertex scope-45 -> Tez vertex group scope-70,Tez vertex group scope-70,Tez vertex group scope-71,Tez vertex group scope-71, +Tez vertex scope-56 -> Tez vertex group scope-70,Tez vertex group scope-71, +Tez vertex group scope-70 +Tez vertex group scope-71 -Tez vertex scope-40 +Tez vertex scope-45 # Plan on vertex -1-2: Split - scope-57 +1-2: Split - scope-69 | | -| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14 +| a2: Split - scope-68 +| | | +| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17 +| | | +| | 1-3: Split - scope-72 +| | | | +| | | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-77 -> scope-38 +| | | | +| | | |---d: Filter[bag] - scope-73 +| | | | | +| | | | Greater Than[boolean] - scope-76 +| | | | | +| | | | |---Project[int][0] - scope-74 +| | | | | +| | | | |---Constant(500) - scope-75 +| | | | +| | | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-83 -> scope-44 +| | | | +| | | |---e: Filter[bag] - scope-78 +| | | | | +| | | | Not[boolean] - scope-82 +| | | | | +| | | | |---Greater Than[boolean] - scope-81 +| | | | | +| | | | |---Project[int][0] - scope-79 +| | | | | +| | | | |---Constant(500) - scope-80 | | | |---a2: Filter[bag] - scope-9 | | | @@ -25,37 +51,37 @@ Tez vertex scope-40 | | | | | |---Constant(100) - scope-11 | | -| 1-3: Split - scope-60 +| 1-3: Split - scope-84 | | | -| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-65 +| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-89 -> scope-38 | | | -| | |---d: Filter[bag] - scope-61 +| | |---d: Filter[bag] - scope-85 | | | | -| | | Greater Than[boolean] - scope-64 +| | | Greater Than[boolean] - scope-88 | | | | -| | | |---Project[int][0] - scope-62 +| | | |---Project[int][0] - scope-86 | | | | -| | | |---Constant(500) - scope-63 +| | | |---Constant(500) - scope-87 | | | -| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-71 +| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-95 -> scope-44 | | | -| | |---e: Filter[bag] - scope-66 +| | |---e: Filter[bag] - scope-90 | | | | -| | | Not[boolean] - scope-70 +| | | Not[boolean] - scope-94 | | | | -| | | |---Greater Than[boolean] - scope-69 +| | | |---Greater Than[boolean] - scope-93 | | | | -| | | |---Project[int][0] - scope-67 +| | | |---Project[int][0] - scope-91 | | | | -| | | |---Constant(500) - scope-68 +| | | |---Constant(500) - scope-92 | | -| |---a1: Filter[bag] - scope-15 +| |---a1: Filter[bag] - scope-18 | | | -| | Greater Than[boolean] - scope-18 +| | Greater Than[boolean] - scope-21 | | | -| | |---Project[int][0] - scope-16 +| | |---Project[int][0] - scope-19 | | | -| | |---Constant(100) - scope-17 +| | |---Constant(100) - scope-20 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -68,44 +94,44 @@ Tez vertex scope-40 | |---Project[bytearray][1] - scope-4 | |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 -Tez vertex scope-46 +Tez vertex scope-56 # Plan on vertex -1-3: Split - scope-72 +1-3: Split - scope-96 | | -| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-77 +| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-101 -> scope-38 | | -| |---d: Filter[bag] - scope-73 +| |---d: Filter[bag] - scope-97 | | | -| | Greater Than[boolean] - scope-76 +| | Greater Than[boolean] - scope-100 | | | -| | |---Project[int][0] - scope-74 +| | |---Project[int][0] - scope-98 | | | -| | |---Constant(500) - scope-75 +| | |---Constant(500) - scope-99 | | -| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-83 +| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-107 -> scope-44 | | -| |---e: Filter[bag] - scope-78 +| |---e: Filter[bag] - scope-102 | | | -| | Not[boolean] - scope-82 +| | Not[boolean] - scope-106 | | | -| | |---Greater Than[boolean] - scope-81 +| | |---Greater Than[boolean] - scope-105 | | | -| | |---Project[int][0] - scope-79 +| | |---Project[int][0] - scope-103 | | | -| | |---Constant(500) - scope-80 +| | |---Constant(500) - scope-104 | -|---c: New For Each(false,false)[bag] - scope-26 +|---c: New For Each(false,false)[bag] - scope-31 | | - | Cast[int] - scope-21 + | Cast[int] - scope-26 | | - | |---Project[bytearray][1] - scope-20 + | |---Project[bytearray][1] - scope-25 | | - | Cast[chararray] - scope-24 + | Cast[chararray] - scope-29 | | - | |---Project[bytearray][0] - scope-23 + | |---Project[bytearray][0] - scope-28 | - |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19 -Tez vertex group scope-58 <- [scope-40, scope-46] -> null + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24 +Tez vertex group scope-70 <- [scope-45, scope-45, scope-56] -> null # No plan on vertex group -Tez vertex group scope-59 <- [scope-40, scope-46] -> null +Tez vertex group scope-71 <- [scope-45, scope-45, scope-56] -> null # No plan on vertex group Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original) +++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Tue Apr 7 23:21:01 2015 @@ -120,6 +120,68 @@ public class TestTezCompiler { } @Test + public void testSelfJoin() throws Exception { + String query = + "a = load 'file:///tmp/input1' as (x:int, y:int);" + + "b = filter a by x < 5;" + + "c = filter a by x == 10;" + + "d = filter a by x > 10;" + + "e = join b by x, c by x, d by x;" + + "store e into 'file:///tmp/output';"; + + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-1.gld"); + } + + @Test + public void testSelfJoinSkewed() throws Exception { + String query = + "a = load 'file:///tmp/input1' as (x:int, y:int);" + + "b = filter a by x < 5;" + + "c = filter a by x == 10;" + + "d = join b by x, c by x using 'skewed';" + + "store d into 'file:///tmp/output';"; + + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-2.gld"); + } + + @Test + public void testSelfJoinReplicated() throws Exception { + String query = + "a = load 'file:///tmp/input1' as (x:int, y:int);" + + "b = filter a by x < 5;" + + "c = filter a by x == 10;" + + "d = filter a by x > 10;" + + "e = join b by x, c by x, d by x using 'replicated';" + + "store e into 'file:///tmp/output';"; + + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-3.gld"); + } + + @Test + public void testCross() throws Exception { + String query = + "a = load 'file:///tmp/input1' as (x:int, y:int);" + + "b = load 'file:///tmp/input2' as (x:int, z:int);" + + "c = cross a, b;" + + "d = foreach c generate a::x as x, y, z;" + + "store d into 'file:///tmp/output';"; + + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld"); + } + + @Test + public void testSelfCross() throws Exception { + String query = + "a = load 'file:///tmp/input1' as (x:int, y:int);" + + "b = filter a by x < 5;" + + "c = filter a by x == 10;" + + "d = cross b, c;" + + "store d into 'file:///tmp/output';"; + + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld"); + } + + @Test public void testSkewedJoin() throws Exception { String query = "a = load 'file:///tmp/input1' as (x:int, y:int);" + @@ -370,6 +432,40 @@ public class TestTezCompiler { } @Test + public void testMultiQueryScalar() throws Exception { + String query = + "a = load 'file:///tmp/input' as (x:int, y:int);" + + "b = group a by x;" + + "c = foreach b generate group, COUNT(a) as cnt;" + + "SPLIT a into d if (2 * c.cnt) < y, e OTHERWISE;" + + "store d into 'file:///tmp/output1';" + + "store e into 'file:///tmp/output2';"; + + setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6-OPTOFF.gld"); + } + + @Test + public void testMultiQueryMultipleReplicateJoin() throws Exception { + String query = + "a = load 'file:///tmp/input' as (x:int, y:int);" + + "b = load 'file:///tmp/input' as (x:int, y:int);" + + "c = join a by $0, b by $0 using 'replicated';" + + "d = join a by $1, b by $1 using 'replicated';" + + "e = union c,d;" + + "store e into 'file:///tmp/output';"; + + setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7-OPTOFF.gld"); + } + + @Test public void testUnionStore() throws Exception { String query = "a = load 'file:///tmp/input' as (x:int, y:chararray);" + @@ -500,7 +596,7 @@ public class TestTezCompiler { "a = load 'file:///tmp/input' as (x:int, y:chararray);" + "b = load 'file:///tmp/input' as (y:chararray, x:int);" + "split a into a1 if x > 100, a2 otherwise;" + - "c = union onschema a1, b;" + + "c = union onschema a1, a2, b;" + "split c into d if x > 500, e otherwise;" + "store a2 into 'file:///tmp/output/a2';" + "store d into 'file:///tmp/output/d';" + @@ -531,8 +627,7 @@ public class TestTezCompiler { run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld"); } - //TODO: union followed by union followed by store does not work. - //@Test + @Test public void testUnionUnionStore() throws Exception { String query = "a = load 'file:///tmp/input' as (x:int, y:chararray);" + @@ -550,6 +645,94 @@ public class TestTezCompiler { } @Test + public void testMultipleUnionSplitJoin() throws Exception { + String query = + "a = load 'file:///tmp/input' as (x:int, y:chararray);" + + "b = filter a by x == 2;" + + "b1 = foreach b generate *;" + + "b2 = foreach b generate *;" + + "b3 = union onschema b1, b2;" + + "c = filter a by x == 3;" + + "c1 = foreach c generate y, x;" + + "c2 = foreach c generate y, x;" + + "c3 = union c1, c2;" + + "a1 = union onschema b3, c3;" + + "d = load 'file:///tmp/input1' as (x:int, z:chararray);" + + "e = join a1 by x, d by x;" + + "store e into 'file:///tmp/output';"; + + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld"); + } + + @Test + public void testUnionSplitReplicateJoin() throws Exception { + String query = + "a = load 'file:///tmp/input' as (x:int, y:chararray);" + + "b = filter a by x == 2;" + + "c = union onschema a, b;" + + "d = load 'file:///tmp/input1' as (x:int, z:chararray);" + + "e = join c by x, d by x using 'replicated';" + + "store e into 'file:///tmp/output';"; + + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13-OPTOFF.gld"); + + query = + "a = load 'file:///tmp/input' as (x:int, y:chararray);" + + "b = filter a by x == 2;" + + "c = union onschema a, b;" + + "d = load 'file:///tmp/input1' as (x:int, z:chararray);" + + "e = join d by x, c by x using 'replicated';" + + "store e into 'file:///tmp/output';"; + + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14-OPTOFF.gld"); + } + + @Test + public void testUnionSplitSkewedJoin() throws Exception { + String query = + "a = load 'file:///tmp/input' as (x:int, y:chararray);" + + "b = filter a by x == 2;" + + "c = union onschema a, b;" + + "d = load 'file:///tmp/input1' as (x:int, z:chararray);" + + "e = join c by x, d by x using 'skewed';" + + "store e into 'file:///tmp/output';"; + + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15-OPTOFF.gld"); + + query = + "a = load 'file:///tmp/input' as (x:int, y:chararray);" + + "b = filter a by x == 2;" + + "c = union onschema a, b;" + + "d = load 'file:///tmp/input1' as (x:int, z:chararray);" + + "e = join d by x, c by x using 'skewed';" + + "store e into 'file:///tmp/output';"; + + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld"); + resetScope(); + setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false); + run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld"); + } + + @Test public void testRank() throws Exception { String query = "a = load 'file:///tmp/input1' as (x:int, y:int);" +
