Author: rohini
Date: Fri Apr 17 16:18:38 2015
New Revision: 1674348
URL: http://svn.apache.org/r1674348
Log:
PIG-4503: [Pig on Tez] NPE in UnionOptimizer with multiple levels of union
(rohini)
Added:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18.gld
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld
pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1674348&r1=1674347&r2=1674348&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr 17 16:18:38 2015
@@ -62,6 +62,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4503: [Pig on Tez] NPE in UnionOptimizer with multiple levels of union
(rohini)
+
PIG-4509: [Pig on Tez] Unassigned applications not killed on shutdown (rohini)
PIG-4508: [Pig on Tez] PigProcessor check for commit only on MROutput (rohini)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java?rev=1674348&r1=1674347&r2=1674348&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
Fri Apr 17 16:18:38 2015
@@ -153,13 +153,15 @@ public class UnionOptimizer extends TezO
TezOperator existingVertexGroup = null;
if (successors != null) {
for (TezOperator succ : successors) {
- if (succ.isVertexGroup() &&
succ.getVertexGroupInfo().getSFile().equals(unionStoreOutputs.get(i).getSFile()))
{
+ if (succ.isVertexGroup() &&
unionStoreOutputs.get(i).getSFile().equals(succ.getVertexGroupInfo().getSFile()))
{
existingVertexGroup = succ;
}
}
}
if (existingVertexGroup != null) {
storeVertexGroupOps[i] = existingVertexGroup;
+
existingVertexGroup.getVertexGroupMembers().remove(unionOp.getOperatorKey());
+
existingVertexGroup.getVertexGroupInfo().removeInput(unionOp.getOperatorKey());
} else {
storeVertexGroupOps[i] = new
TezOperator(OperatorKey.genOpKey(scope));
storeVertexGroupOps[i].setVertexGroupInfo(new
VertexGroupInfo(unionStoreOutputs.get(i)));
@@ -471,8 +473,8 @@ public class UnionOptimizer extends TezO
TezOperator succOpVertexGroup = null;
for (TezOperator succ : successors) {
if (succ.isVertexGroup()
- && succ.getVertexGroupInfo().getOutput()
- .equals(succOp.getOperatorKey().toString())) {
+ && succOp.getOperatorKey().toString()
+
.equals(succ.getVertexGroupInfo().getOutput())) {
succOpVertexGroup = succ;
break;
}
Modified:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld?rev=1674348&r1=1674347&r2=1674348&view=diff
==============================================================================
---
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld
(original)
+++
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld
Fri Apr 17 16:18:38 2015
@@ -4,16 +4,16 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-31 -> Tez vertex scope-33,
-Tez vertex scope-32 -> Tez vertex scope-33,
-Tez vertex scope-33 -> Tez vertex scope-38,
-Tez vertex scope-37 -> Tez vertex scope-38,
-Tez vertex scope-38 -> Tez vertex scope-42,
-Tez vertex scope-42
+Tez vertex scope-37 -> Tez vertex scope-39,
+Tez vertex scope-38 -> Tez vertex scope-39,
+Tez vertex scope-39 -> Tez vertex scope-44,
+Tez vertex scope-43 -> Tez vertex scope-44,
+Tez vertex scope-44 -> Tez vertex scope-53,
+Tez vertex scope-53
-Tez vertex scope-31
+Tez vertex scope-37
# Plan on vertex
-POValueOutputTez - scope-35 -> [scope-33]
+POValueOutputTez - scope-41 -> [scope-39]
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -26,9 +26,9 @@ POValueOutputTez - scope-35 -> [scope-3
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-32
+Tez vertex scope-38
# Plan on vertex
-POValueOutputTez - scope-36 -> [scope-33]
+POValueOutputTez - scope-42 -> [scope-39]
|
|---c: New For Each(false,false)[bag] - scope-15
| |
@@ -41,14 +41,14 @@ POValueOutputTez - scope-36 -> [scope-3
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex scope-33
+Tez vertex scope-39
# Plan on vertex
-POValueOutputTez - scope-40 -> [scope-38]
+POValueOutputTez - scope-46 -> [scope-44]
|
-|---POShuffledValueInputTez - scope-34 <- [scope-31, scope-32]
-Tez vertex scope-37
+|---POShuffledValueInputTez - scope-40 <- [scope-37, scope-38]
+Tez vertex scope-43
# Plan on vertex
-POValueOutputTez - scope-41 -> [scope-38]
+POValueOutputTez - scope-47 -> [scope-44]
|
|---d: New For Each(false,false)[bag] - scope-24
| |
@@ -61,15 +61,19 @@ POValueOutputTez - scope-41 -> [scope-3
| |---Project[bytearray][1] - scope-21
|
|---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-17
-Tez vertex scope-38
+Tez vertex scope-44
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-28 -> scope-42
+e: Split - scope-54
+| |
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-29
| |
-| Project[int][0] - scope-29
+| f: Local Rearrange[tuple]{int}(false) - scope-34 -> scope-53
+| | |
+| | Project[int][0] - scope-35
|
-|---POShuffledValueInputTez - scope-39 <- [scope-33, scope-37]
-Tez vertex scope-42
+|---POShuffledValueInputTez - scope-45 <- [scope-39, scope-43]
+Tez vertex scope-53
# Plan on vertex
-f: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-30
+f: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-36
|
-|---f: Package(Packager)[tuple]{int} - scope-27
+|---f: Package(Packager)[tuple]{int} - scope-33
Modified:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld?rev=1674348&r1=1674347&r2=1674348&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
(original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
Fri Apr 17 16:18:38 2015
@@ -4,17 +4,22 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-37 -> Tez vertex group scope-43,
-Tez vertex scope-31 -> Tez vertex group scope-43,
-Tez vertex scope-32 -> Tez vertex group scope-43,
-Tez vertex group scope-43 -> Tez vertex scope-42,
-Tez vertex scope-42
+Tez vertex scope-43 -> Tez vertex group scope-55,Tez vertex group
scope-56,
+Tez vertex scope-37 -> Tez vertex group scope-55,Tez vertex group
scope-56,
+Tez vertex scope-38 -> Tez vertex group scope-55,Tez vertex group
scope-56,
+Tez vertex group scope-56 -> Tez vertex scope-53,
+Tez vertex scope-53
+Tez vertex group scope-55
-Tez vertex scope-37
+Tez vertex scope-43
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-46 -> scope-42
+e: Split - scope-61
+| |
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-62
-> scope-29
| |
-| Project[int][0] - scope-47
+| f: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-53
+| | |
+| | Project[int][0] - scope-64
|
|---d: New For Each(false,false)[bag] - scope-24
| |
@@ -27,11 +32,15 @@ f: Local Rearrange[tuple]{int}(false) -
| |---Project[bytearray][1] - scope-21
|
|---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-17
-Tez vertex scope-31
+Tez vertex scope-37
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-49 -> scope-42
+e: Split - scope-66
| |
-| Project[int][0] - scope-50
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-67
-> scope-29
+| |
+| f: Local Rearrange[tuple]{int}(false) - scope-68 -> scope-53
+| | |
+| | Project[int][0] - scope-69
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -44,11 +53,15 @@ f: Local Rearrange[tuple]{int}(false) -
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-32
+Tez vertex scope-38
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-51 -> scope-42
+e: Split - scope-70
+| |
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-71
-> scope-29
| |
-| Project[int][0] - scope-52
+| f: Local Rearrange[tuple]{int}(false) - scope-72 -> scope-53
+| | |
+| | Project[int][0] - scope-73
|
|---c: New For Each(false,false)[bag] - scope-15
| |
@@ -61,10 +74,12 @@ f: Local Rearrange[tuple]{int}(false) -
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex group scope-43 <- [scope-37, scope-31, scope-32] ->
scope-42
+Tez vertex group scope-56 <- [scope-43, scope-37, scope-38] ->
scope-53
# No plan on vertex group
-Tez vertex scope-42
+Tez vertex scope-53
# Plan on vertex
-f: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-30
+f: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-36
|
-|---f: Package(Packager)[tuple]{int} - scope-27
+|---f: Package(Packager)[tuple]{int} - scope-33
+Tez vertex group scope-55 <- [scope-43, scope-37, scope-38] ->
null
+# No plan on vertex group
Modified:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld?rev=1674348&r1=1674347&r2=1674348&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld
(original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld
Fri Apr 17 16:18:38 2015
@@ -54,5 +54,5 @@ e: Store(file:///tmp/output:org.apache.p
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex group scope-38 <- [scope-29, scope-33, scope-27,
scope-28] -> null
+Tez vertex group scope-38 <- [scope-33, scope-27, scope-28] ->
null
# No plan on vertex group
Added:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17-OPTOFF.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17-OPTOFF.gld?rev=1674348&view=auto
==============================================================================
---
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17-OPTOFF.gld
(added)
+++
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17-OPTOFF.gld
Fri Apr 17 16:18:38 2015
@@ -0,0 +1,87 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-44 -> Tez vertex scope-45,Tez vertex scope-46,
+Tez vertex scope-45 -> Tez vertex scope-47,
+Tez vertex scope-46 -> Tez vertex scope-47,
+Tez vertex scope-47
+
+Tez vertex scope-44
+# Plan on vertex
+POValueOutputTez - scope-51 -> [scope-45, scope-46]
+|
+|---d: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-45
+# Plan on vertex
+POValueOutputTez - scope-49 -> [scope-47]
+|
+|---e: Filter[bag] - scope-17
+ | |
+ | Equal To[boolean] - scope-22
+ | |
+ | |---Project[int][0] - scope-18
+ | |
+ |
|---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[int]
- scope-21
+ | |
+ | |---Constant(0) - scope-19
+ |
+ |---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) -
scope-0
+Tez vertex scope-46
+# Plan on vertex
+POValueOutputTez - scope-50 -> [scope-47]
+|
+|---c: New For Each(false,false)[bag] - scope-41
+ | |
+ | Project[int][1] - scope-37
+ | |
+ | Project[chararray][0] - scope-39
+ |
+ |---Filter[bag] - scope-31
+ | |
+ | Equal To[boolean] - scope-36
+ | |
+ | |---Project[int][1] - scope-32
+ | |
+ |
|---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[int]
- scope-35
+ | |
+ | |---Constant(0) - scope-33
+ |
+ |---b: New For Each(false,false)[bag] - scope-30
+ | |
+ | Cast[chararray] - scope-25
+ | |
+ | |---Project[bytearray][0] - scope-24
+ | |
+ | Cast[int] - scope-28
+ | |
+ | |---Project[bytearray][1] - scope-27
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) -
scope-23
+Tez vertex scope-47
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-43
+|
+|---POShuffledValueInputTez - scope-48 <- [scope-45, scope-46]
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17.gld?rev=1674348&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17.gld
(added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17.gld
Fri Apr 17 16:18:38 2015
@@ -0,0 +1,84 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-44 -> Tez vertex scope-45,Tez vertex scope-46,
+Tez vertex scope-45 -> Tez vertex group scope-52,
+Tez vertex scope-46 -> Tez vertex group scope-52,
+Tez vertex group scope-52
+
+Tez vertex scope-44
+# Plan on vertex
+POValueOutputTez - scope-51 -> [scope-45, scope-46]
+|
+|---d: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-45
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-53
-> scope-43
+|
+|---e: Filter[bag] - scope-17
+ | |
+ | Equal To[boolean] - scope-22
+ | |
+ | |---Project[int][0] - scope-18
+ | |
+ |
|---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[int]
- scope-21
+ | |
+ | |---Constant(0) - scope-19
+ |
+ |---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) -
scope-0
+Tez vertex scope-46
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-54
-> scope-43
+|
+|---c: New For Each(false,false)[bag] - scope-41
+ | |
+ | Project[int][1] - scope-37
+ | |
+ | Project[chararray][0] - scope-39
+ |
+ |---Filter[bag] - scope-31
+ | |
+ | Equal To[boolean] - scope-36
+ | |
+ | |---Project[int][1] - scope-32
+ | |
+ |
|---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[int]
- scope-35
+ | |
+ | |---Constant(0) - scope-33
+ |
+ |---b: New For Each(false,false)[bag] - scope-30
+ | |
+ | Cast[chararray] - scope-25
+ | |
+ | |---Project[bytearray][0] - scope-24
+ | |
+ | Cast[int] - scope-28
+ | |
+ | |---Project[bytearray][1] - scope-27
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) -
scope-23
+Tez vertex group scope-52 <- [scope-45, scope-46] -> null
+# No plan on vertex group
Added:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18-OPTOFF.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18-OPTOFF.gld?rev=1674348&view=auto
==============================================================================
---
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18-OPTOFF.gld
(added)
+++
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18-OPTOFF.gld
Fri Apr 17 16:18:38 2015
@@ -0,0 +1,71 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-33 -> Tez vertex scope-35,
+Tez vertex scope-34 -> Tez vertex scope-35,
+Tez vertex scope-35 -> Tez vertex scope-39,
+Tez vertex scope-39
+
+Tez vertex scope-33
+# Plan on vertex
+POValueOutputTez - scope-37 -> [scope-35]
+|
+|---a: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-34
+# Plan on vertex
+POValueOutputTez - scope-38 -> [scope-35]
+|
+|---c: New For Each(false,false)[bag] - scope-23
+ | |
+ | Cast[int] - scope-18
+ | |
+ | |---Project[bytearray][1] - scope-17
+ | |
+ | Cast[chararray] - scope-21
+ | |
+ | |---Project[bytearray][0] - scope-20
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-16
+Tez vertex scope-35
+# Plan on vertex
+POValueOutputTez - scope-40 -> [scope-39]
+|
+|---POShuffledValueInputTez - scope-36 <- [scope-33, scope-34]
+Tez vertex scope-39
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-32
+|
+|---e: Filter[bag] - scope-26
+ | |
+ | Equal To[boolean] - scope-31
+ | |
+ | |---Project[int][0] - scope-27
+ | |
+ |
|---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[int]
- scope-30
+ | |
+ | |---Constant(0) - scope-28
+ |
+ |---d: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-0
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18.gld?rev=1674348&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18.gld
(added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18.gld
Fri Apr 17 16:18:38 2015
@@ -0,0 +1,68 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-33 -> Tez vertex group scope-41,
+Tez vertex scope-34 -> Tez vertex group scope-41,
+Tez vertex group scope-41 -> Tez vertex scope-39,
+Tez vertex scope-39
+
+Tez vertex scope-33
+# Plan on vertex
+POValueOutputTez - scope-42 -> [scope-39]
+|
+|---a: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-34
+# Plan on vertex
+POValueOutputTez - scope-43 -> [scope-39]
+|
+|---c: New For Each(false,false)[bag] - scope-23
+ | |
+ | Cast[int] - scope-18
+ | |
+ | |---Project[bytearray][1] - scope-17
+ | |
+ | Cast[chararray] - scope-21
+ | |
+ | |---Project[bytearray][0] - scope-20
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-16
+Tez vertex group scope-41 <- [scope-33, scope-34] ->
scope-39
+# No plan on vertex group
+Tez vertex scope-39
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-32
+|
+|---e: Filter[bag] - scope-26
+ | |
+ | Equal To[boolean] - scope-31
+ | |
+ | |---Project[int][0] - scope-27
+ | |
+ |
|---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[int]
- scope-30
+ | |
+ | |---Constant(0) - scope-28
+ |
+ |---d: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-0
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1674348&r1=1674347&r2=1674348&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Fri Apr 17 16:18:38
2015
@@ -618,13 +618,15 @@ public class TestTezCompiler {
"d = load 'file:///tmp/input1' as (x:int, y:chararray);" +
"e = union onschema c, d;" +
"f = group e by x;" +
- "store f into 'file:///tmp/output';";
+ "store e into 'file:///tmp/output1';" +
+ "store f into 'file:///tmp/output2';";
setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld");
resetScope();
setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld");
+
}
@Test
@@ -733,6 +735,38 @@ public class TestTezCompiler {
}
@Test
+ public void testUnionScalar() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = filter c by x == d.x;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-17-OPTOFF.gld");
+
+ query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = filter d by x == c.x;" +
+ "store e into 'file:///tmp/output';";
+
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-18-OPTOFF.gld");
+ }
+
+ @Test
public void testRank() throws Exception {
String query =
"a = load 'file:///tmp/input1' as (x:int, y:int);" +
