Author: rohini
Date: Mon Oct 31 15:56:41 2016
New Revision: 1767320
URL: http://svn.apache.org/viewvc?rev=1767320&view=rev
Log:
PIG-5033: MultiQueryOptimizerTez creates bad plan with union, split and FRJoin
(rohini,tmwoordruff via rohini)
Added:
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10-OPTOFF.gld
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10.gld
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9-OPTOFF.gld
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9.gld
Modified:
pig/branches/branch-0.16/CHANGES.txt
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
pig/branches/branch-0.16/test/e2e/pig/tests/multiquery.conf
pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/branches/branch-0.16/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1767320&r1=1767319&r2=1767320&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Mon Oct 31 15:56:41 2016
@@ -32,6 +32,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5033: MultiQueryOptimizerTez creates bad plan with union, split and FRJoin
(rohini,tmwoordruff via rohini)
+
PIG-5041: RoundRobinPartitioner is not deterministic when order of input
records change (rohini)
PIG-5040: Order by and CROSS partitioning is not deterministic due to usage of
Random (rohini)
Modified:
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java?rev=1767320&r1=1767319&r2=1767320&view=diff
==============================================================================
---
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
(original)
+++
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
Mon Oct 31 15:56:41 2016
@@ -153,6 +153,8 @@ public class MultiQueryOptimizerTez exte
}
}
if (getPlan().getSuccessors(successor) != null) {
+ nonPackageInputSuccessors.clear();
+ toMergeSuccessors.clear();
for (TezOperator succSuccessor :
getPlan().getSuccessors(successor)) {
if (succSuccessor.isUnion()) {
if (!(unionOptimizerOn &&
@@ -171,7 +173,13 @@ public class MultiQueryOptimizerTez exte
continue;
}
}
- toMergeSuccessors.add(succSuccessor);
+ if
(TezCompilerUtil.isNonPackageInput(successor.getOperatorKey().toString(),
succSuccessor)) {
+ // Output goes to scalar or POFRJoinTez in
the union operator
+ // We need to ensure it is the only one to
avoid parallel edges
+ canMerge = canMerge ?
nonPackageInputSuccessors.add(succSuccessor) : false;
+ } else {
+ toMergeSuccessors.add(succSuccessor);
+ }
List<TezOperator> unionSuccessors =
getPlan().getSuccessors(succSuccessor);
if (unionSuccessors != null) {
for (TezOperator unionSuccessor :
unionSuccessors) {
Modified:
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java?rev=1767320&r1=1767319&r2=1767320&view=diff
==============================================================================
---
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
(original)
+++
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
Mon Oct 31 15:56:41 2016
@@ -36,6 +36,7 @@ import org.apache.pig.backend.hadoop.exe
import
org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import
org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POFRJoinTez;
import
org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez;
import
org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez;
import
org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
@@ -198,8 +199,8 @@ public class TezCompilerUtil {
public static boolean isNonPackageInput(String inputKey, TezOperator
tezOp) throws PlanException {
try {
- List<TezInput> inputs =
PlanHelper.getPhysicalOperators(tezOp.plan, TezInput.class);
- for (TezInput input : inputs) {
+ List<POFRJoinTez> inputs =
PlanHelper.getPhysicalOperators(tezOp.plan, POFRJoinTez.class);
+ for (POFRJoinTez input : inputs) {
if (ArrayUtils.contains(input.getTezInputs(), inputKey)) {
return true;
}
Modified: pig/branches/branch-0.16/test/e2e/pig/tests/multiquery.conf
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/e2e/pig/tests/multiquery.conf?rev=1767320&r1=1767319&r2=1767320&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/e2e/pig/tests/multiquery.conf (original)
+++ pig/branches/branch-0.16/test/e2e/pig/tests/multiquery.conf Mon Oct 31
15:56:41 2016
@@ -728,6 +728,20 @@ b = union a1, a2;
c = rank b by name ASC, age DESC DENSE;
store c into ':OUTPATH:';\,
},
+ {
+ # Union + Split + Two replicate join
+ 'num' => 12,
+ 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as
(name, age, gpa);
+a1 = filter a by gpa is null or gpa <= 3.9;
+a2 = filter a by gpa < 2;
+b = union a1, a2;
+c = load ':INPATH:/singlefile/voternulltab10k' as (name, age, registration,
contributions);
+c1 = filter c by age < 30;
+c2 = filter c by age > 50;
+d = join b by name, c1 by name using 'replicated';
+e = join d by b::name, c2 by name using 'replicated';
+store e into ':OUTPATH:';\,
+ }
] # end of tests
},
Added:
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10-OPTOFF.gld
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10-OPTOFF.gld?rev=1767320&view=auto
==============================================================================
---
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10-OPTOFF.gld
(added)
+++
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10-OPTOFF.gld
Mon Oct 31 15:56:41 2016
@@ -0,0 +1,109 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-56 -> Tez vertex scope-58,Tez vertex scope-60,
+Tez vertex scope-58 -> Tez vertex scope-62,
+Tez vertex scope-60 -> Tez vertex scope-62,
+Tez vertex scope-66 -> Tez vertex scope-68,Tez vertex scope-70,
+Tez vertex scope-68 -> Tez vertex scope-62,
+Tez vertex scope-70 -> Tez vertex scope-62,
+Tez vertex scope-62
+
+Tez vertex scope-56
+# Plan on vertex
+POValueOutputTez - scope-57 -> [scope-58, scope-60]
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-58
+# Plan on vertex
+POValueOutputTez - scope-64 -> [scope-62]
+|
+|---POValueInputTez - scope-59 <- scope-56
+Tez vertex scope-60
+# Plan on vertex
+POValueOutputTez - scope-65 -> [scope-62]
+|
+|---b: Filter[bag] - scope-13
+ | |
+ | Equal To[boolean] - scope-16
+ | |
+ | |---Project[int][0] - scope-14
+ | |
+ | |---Constant(2) - scope-15
+ |
+ |---POValueInputTez - scope-61 <- scope-56
+Tez vertex scope-66
+# Plan on vertex
+POValueOutputTez - scope-67 -> [scope-68, scope-70]
+|
+|---c: New For Each(false,false)[bag] - scope-25
+ | |
+ | Cast[int] - scope-20
+ | |
+ | |---Project[bytearray][0] - scope-19
+ | |
+ | Cast[int] - scope-23
+ | |
+ | |---Project[bytearray][1] - scope-22
+ |
+ |---c: Load(file:///tmp/input3:org.apache.pig.builtin.PigStorage) -
scope-18
+Tez vertex scope-68
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-38 -> scope-62
+| |
+| Project[int][0] - scope-34
+|
+|---e: Filter[bag] - scope-29
+ | |
+ | Less Than[boolean] - scope-32
+ | |
+ | |---Project[int][1] - scope-30
+ | |
+ | |---Constant(2) - scope-31
+ |
+ |---POValueInputTez - scope-69 <- scope-66
+Tez vertex scope-70
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-52 -> scope-62
+| |
+| Project[int][0] - scope-48
+|
+|---f: Filter[bag] - scope-43
+ | |
+ | Greater Than[boolean] - scope-46
+ | |
+ | |---Project[int][1] - scope-44
+ | |
+ | |---Constant(5) - scope-45
+ |
+ |---POValueInputTez - scope-71 <- scope-66
+Tez vertex scope-62
+# Plan on vertex
+h: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-55
+|
+|---h: FRJoin[tuple] - scope-49 <- scope-70
+ | |
+ | Project[int][0] - scope-47
+ | |
+ | Project[int][0] - scope-48
+ |
+ |---g: FRJoin[tuple] - scope-35 <- scope-68
+ | |
+ | Project[int][0] - scope-33
+ | |
+ | Project[int][0] - scope-34
+ |
+ |---POShuffledValueInputTez - scope-63 <- [scope-58, scope-60]
Added:
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10.gld
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10.gld?rev=1767320&view=auto
==============================================================================
---
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10.gld
(added)
+++
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10.gld
Mon Oct 31 15:56:41 2016
@@ -0,0 +1,105 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-66 -> Tez vertex scope-56,Tez vertex scope-70,
+Tez vertex scope-70 -> Tez vertex scope-56,
+Tez vertex scope-56
+
+Tez vertex scope-66
+# Plan on vertex
+c: Split - scope-73
+| |
+| Local Rearrange[tuple]{int}(false) - scope-38 -> scope-56
+| | |
+| | Project[int][0] - scope-34
+| |
+| |---e: Filter[bag] - scope-29
+| | |
+| | Less Than[boolean] - scope-32
+| | |
+| | |---Project[int][1] - scope-30
+| | |
+| | |---Constant(2) - scope-31
+| |
+| POValueOutputTez - scope-67 -> [scope-70]
+|
+|---c: New For Each(false,false)[bag] - scope-25
+ | |
+ | Cast[int] - scope-20
+ | |
+ | |---Project[bytearray][0] - scope-19
+ | |
+ | Cast[int] - scope-23
+ | |
+ | |---Project[bytearray][1] - scope-22
+ |
+ |---c: Load(file:///tmp/input3:org.apache.pig.builtin.PigStorage) -
scope-18
+Tez vertex scope-70
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-52 -> scope-56
+| |
+| Project[int][0] - scope-48
+|
+|---f: Filter[bag] - scope-43
+ | |
+ | Greater Than[boolean] - scope-46
+ | |
+ | |---Project[int][1] - scope-44
+ | |
+ | |---Constant(5) - scope-45
+ |
+ |---POValueInputTez - scope-71 <- scope-66
+Tez vertex scope-56
+# Plan on vertex
+a: Split - scope-72
+| |
+| h: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) -
scope-90 -> scope-55
+| |
+| |---h: FRJoin[tuple] - scope-82 <- scope-70
+| | |
+| | Project[int][0] - scope-83
+| | |
+| | Project[int][0] - scope-84
+| |
+| |---g: FRJoin[tuple] - scope-74 <- scope-66
+| | |
+| | Project[int][0] - scope-75
+| | |
+| | Project[int][0] - scope-76
+| |
+| h: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) -
scope-107 -> scope-55
+| |
+| |---h: FRJoin[tuple] - scope-99 <- scope-70
+| | |
+| | Project[int][0] - scope-100
+| | |
+| | Project[int][0] - scope-101
+| |
+| |---g: FRJoin[tuple] - scope-91 <- scope-66
+| | |
+| | Project[int][0] - scope-92
+| | |
+| | Project[int][0] - scope-93
+| |
+| |---b: Filter[bag] - scope-13
+| | |
+| | Equal To[boolean] - scope-16
+| | |
+| | |---Project[int][0] - scope-14
+| | |
+| | |---Constant(2) - scope-15
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
Added:
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9-OPTOFF.gld
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9-OPTOFF.gld?rev=1767320&view=auto
==============================================================================
---
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9-OPTOFF.gld
(added)
+++
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9-OPTOFF.gld
Mon Oct 31 15:56:41 2016
@@ -0,0 +1,105 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-55 -> Tez vertex scope-57,
+Tez vertex scope-56 -> Tez vertex scope-57,
+Tez vertex scope-61 -> Tez vertex scope-63,Tez vertex scope-65,
+Tez vertex scope-63 -> Tez vertex scope-57,
+Tez vertex scope-65 -> Tez vertex scope-57,
+Tez vertex scope-57
+
+Tez vertex scope-55
+# Plan on vertex
+POValueOutputTez - scope-59 -> [scope-57]
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-56
+# Plan on vertex
+POValueOutputTez - scope-60 -> [scope-57]
+|
+|---b: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[int] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-61
+# Plan on vertex
+POValueOutputTez - scope-62 -> [scope-63, scope-65]
+|
+|---c: New For Each(false,false)[bag] - scope-24
+ | |
+ | Cast[int] - scope-19
+ | |
+ | |---Project[bytearray][0] - scope-18
+ | |
+ | Cast[int] - scope-22
+ | |
+ | |---Project[bytearray][1] - scope-21
+ |
+ |---c: Load(file:///tmp/input3:org.apache.pig.builtin.PigStorage) -
scope-17
+Tez vertex scope-63
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-37 -> scope-57
+| |
+| Project[int][0] - scope-33
+|
+|---e: Filter[bag] - scope-28
+ | |
+ | Less Than[boolean] - scope-31
+ | |
+ | |---Project[int][1] - scope-29
+ | |
+ | |---Constant(2) - scope-30
+ |
+ |---POValueInputTez - scope-64 <- scope-61
+Tez vertex scope-65
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-51 -> scope-57
+| |
+| Project[int][0] - scope-47
+|
+|---f: Filter[bag] - scope-42
+ | |
+ | Greater Than[boolean] - scope-45
+ | |
+ | |---Project[int][1] - scope-43
+ | |
+ | |---Constant(5) - scope-44
+ |
+ |---POValueInputTez - scope-66 <- scope-61
+Tez vertex scope-57
+# Plan on vertex
+h: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-54
+|
+|---h: FRJoin[tuple] - scope-48 <- scope-65
+ | |
+ | Project[int][0] - scope-46
+ | |
+ | Project[int][0] - scope-47
+ |
+ |---g: FRJoin[tuple] - scope-34 <- scope-63
+ | |
+ | Project[int][0] - scope-32
+ | |
+ | Project[int][0] - scope-33
+ |
+ |---POShuffledValueInputTez - scope-58 <- [scope-55, scope-56]
Added:
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9.gld
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9.gld?rev=1767320&view=auto
==============================================================================
---
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9.gld
(added)
+++
pig/branches/branch-0.16/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9.gld
Mon Oct 31 15:56:41 2016
@@ -0,0 +1,103 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-55 -> Tez vertex scope-57,
+Tez vertex scope-56 -> Tez vertex scope-57,
+Tez vertex scope-61 -> Tez vertex scope-57,Tez vertex scope-65,
+Tez vertex scope-65 -> Tez vertex scope-57,
+Tez vertex scope-57
+
+Tez vertex scope-55
+# Plan on vertex
+POValueOutputTez - scope-59 -> [scope-57]
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-56
+# Plan on vertex
+POValueOutputTez - scope-60 -> [scope-57]
+|
+|---b: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[int] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-61
+# Plan on vertex
+c: Split - scope-67
+| |
+| Local Rearrange[tuple]{int}(false) - scope-37 -> scope-57
+| | |
+| | Project[int][0] - scope-33
+| |
+| |---e: Filter[bag] - scope-28
+| | |
+| | Less Than[boolean] - scope-31
+| | |
+| | |---Project[int][1] - scope-29
+| | |
+| | |---Constant(2) - scope-30
+| |
+| POValueOutputTez - scope-62 -> [scope-65]
+|
+|---c: New For Each(false,false)[bag] - scope-24
+ | |
+ | Cast[int] - scope-19
+ | |
+ | |---Project[bytearray][0] - scope-18
+ | |
+ | Cast[int] - scope-22
+ | |
+ | |---Project[bytearray][1] - scope-21
+ |
+ |---c: Load(file:///tmp/input3:org.apache.pig.builtin.PigStorage) -
scope-17
+Tez vertex scope-65
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-51 -> scope-57
+| |
+| Project[int][0] - scope-47
+|
+|---f: Filter[bag] - scope-42
+ | |
+ | Greater Than[boolean] - scope-45
+ | |
+ | |---Project[int][1] - scope-43
+ | |
+ | |---Constant(5) - scope-44
+ |
+ |---POValueInputTez - scope-66 <- scope-61
+Tez vertex scope-57
+# Plan on vertex
+h: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-54
+|
+|---h: FRJoin[tuple] - scope-48 <- scope-65
+ | |
+ | Project[int][0] - scope-46
+ | |
+ | Project[int][0] - scope-47
+ |
+ |---g: FRJoin[tuple] - scope-34 <- scope-61
+ | |
+ | Project[int][0] - scope-32
+ | |
+ | Project[int][0] - scope-33
+ |
+ |---POShuffledValueInputTez - scope-58 <- [scope-55, scope-56]
Modified: pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezCompiler.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezCompiler.java?rev=1767320&r1=1767319&r2=1767320&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezCompiler.java
(original)
+++ pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezCompiler.java Mon
Oct 31 15:56:41 2016
@@ -632,6 +632,46 @@ public class TestTezCompiler {
}
@Test
+ public void testMultiQueryMultipleReplicateJoinWithUnion() throws
Exception {
+ // Replicate joins are from a split
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, y:int);" +
+ "c = load 'file:///tmp/input3' as (x:int, y:int);" +
+ "d = union a, b;" +
+ "e = filter c by y < 2;" +
+ "f = filter c by y > 5;" +
+ "g = join d by x, e by x using 'replicated';" +
+ "h = join g by d::x, f by x using 'replicated';" +
+ "store h into 'file:///tmp/pigoutput';";
+
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + true);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + false);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-9-OPTOFF.gld");
+
+ // Union is also from a split
+ query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = filter a by x == 2;" +
+ "c = load 'file:///tmp/input3' as (x:int, y:int);" +
+ "d = union a, b;" +
+ "e = filter c by y < 2;" +
+ "f = filter c by y > 5;" +
+ "g = join d by x, e by x using 'replicated';" +
+ "h = join g by d::x, f by x using 'replicated';" +
+ "store h into 'file:///tmp/pigoutput';";
+
+ resetScope();
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + true);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + false);
+ run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-10-OPTOFF.gld");
+ }
+
+ @Test
public void testUnionStore() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:chararray);" +
