Author: rohini
Date: Tue Jan 5 08:26:12 2016
New Revision: 1723016
URL: http://svn.apache.org/viewvc?rev=1723016&view=rev
Log:
PIG-4769: UnionOptimizer hits errors when merging vertex group into split
(rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
pig/trunk/test/e2e/pig/tests/multiquery.conf
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld
pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 5 08:26:12 2016
@@ -77,6 +77,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4769: UnionOptimizer hits errors when merging vertex group into split
(rohini)
+
PIG-4768: EvalFunc reporter is null in Tez (rohini)
PIG-4760: TezDAGStats.convertToHadoopCounters is not used, but impose MR
counter limit (daijy)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
Tue Jan 5 08:26:12 2016
@@ -23,9 +23,11 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -36,7 +38,6 @@ import org.apache.pig.backend.hadoop.exe
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
import
org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.TezOperDependencyParallelismEstimator.TezParallelismFactorVisitor;
import org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor;
-import org.apache.pig.backend.hadoop.executionengine.tez.util.TezCompilerUtil;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.Operator;
import org.apache.pig.impl.plan.OperatorKey;
@@ -509,6 +510,16 @@ public class TezOperator extends Operato
this.vertexGroupStores.put(storeKey, vertexGroupKey);
}
+ public void removeVertexGroupStore(OperatorKey vertexGroupKey) {
+ Iterator<Entry<OperatorKey, OperatorKey>> iter =
vertexGroupStores.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<OperatorKey, OperatorKey> entry = iter.next();
+ if (entry.getValue().equals(vertexGroupKey)) {
+ iter.remove();
+ }
+ }
+ }
+
public Map<OperatorKey, OperatorKey> getVertexGroupStores() {
return this.vertexGroupStores;
}
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
Tue Jan 5 08:26:12 2016
@@ -33,6 +33,7 @@ import org.apache.pig.backend.hadoop.exe
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
import
org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
@@ -219,7 +220,7 @@ public class UnionOptimizer extends TezO
storeVertexGroupOps[i] = new
TezOperator(OperatorKey.genOpKey(scope));
storeVertexGroupOps[i].setVertexGroupInfo(new
VertexGroupInfo(unionStoreOutputs.get(i)));
storeVertexGroupOps[i].getVertexGroupInfo().setSFile(unionStoreOutputs.get(i).getSFile());
-
storeVertexGroupOps[i].setVertexGroupMembers(unionOp.getUnionMembers());
+ storeVertexGroupOps[i].setVertexGroupMembers(new
ArrayList<OperatorKey>(unionOp.getUnionMembers()));
tezPlan.add(storeVertexGroupOps[i]);
}
}
@@ -243,7 +244,7 @@ public class UnionOptimizer extends TezO
outputVertexGroupOps[i] = new
TezOperator(OperatorKey.genOpKey(scope));
outputVertexGroupOps[i].setVertexGroupInfo(new VertexGroupInfo());
outputVertexGroupOps[i].getVertexGroupInfo().setOutput(unionOutputKeys.get(i));
-
outputVertexGroupOps[i].setVertexGroupMembers(unionOp.getUnionMembers());
+ outputVertexGroupOps[i].setVertexGroupMembers(new
ArrayList<OperatorKey>(unionOp.getUnionMembers()));
newOutputKeys[i] =
outputVertexGroupOps[i].getOperatorKey().toString();
tezPlan.add(outputVertexGroupOps[i]);
}
@@ -365,6 +366,17 @@ public class UnionOptimizer extends TezO
succ.getVertexGroupMembers().set(index,
splitPredOp.getOperatorKey());
index =
succ.getVertexGroupMembers().indexOf(unionOp.getOperatorKey());
}
+ // Store vertex group
+ POStore store =
successorVertexGroup.getVertexGroupInfo().getStore();
+ if (store != null) {
+ //Clone changes the operator keys
+ List<POStoreTez> storeOutputs =
PlanHelper.getPhysicalOperators(splitPredOp.plan, POStoreTez.class);
+ for (POStoreTez storeOut : storeOutputs) {
+ if
(storeOut.getOutputKey().equals(store.getOperatorKey().toString())) {
+
splitPredOp.addVertexGroupStore(storeOut.getOperatorKey(),
successorVertexGroup.getOperatorKey());
+ }
+ }
+ }
tezPlan.disconnect(unionOp, successorVertexGroup);
Set<OperatorKey> uniqueVertexGroupMembers = new
HashSet<OperatorKey>(succ.getVertexGroupMembers());
if (uniqueVertexGroupMembers.size() == 1) {
@@ -377,9 +389,15 @@ public class UnionOptimizer extends TezO
actualSuccs.add(succ);
}
+ // Store vertex group
+ if (actualSuccs.isEmpty() && removeSuccessorVertexGroup) {
+
splitPredOp.removeVertexGroupStore(successorVertexGroup.getOperatorKey());
+ tezPlan.remove(successorVertexGroup);
+ }
+
for (TezOperator actualSucc : actualSuccs) {
- TezCompilerUtil.replaceInput(succ, unionOpKey,
splitPredOpKey);
+ TezCompilerUtil.replaceInput(actualSucc, unionOpKey,
splitPredOpKey);
TezEdgeDescriptor edge =
actualSucc.inEdges.remove(unionOp.getOperatorKey());
if (edge == null) {
@@ -393,6 +411,7 @@ public class UnionOptimizer extends TezO
// to SplitOp -> Successor
tezPlan.disconnect(successorVertexGroup,
actualSucc);
tezPlan.remove(successorVertexGroup);
+ TezCompilerUtil.replaceInput(actualSucc,
successorVertexGroup.getOperatorKey().toString(), splitPredOpKey);
} else {
// Changes plan from SplitOp -> Union -> Successor
// to SplitOp -> Successor
Modified: pig/trunk/test/e2e/pig/tests/multiquery.conf
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/multiquery.conf?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/multiquery.conf (original)
+++ pig/trunk/test/e2e/pig/tests/multiquery.conf Tue Jan 5 08:26:12 2016
@@ -671,6 +671,24 @@ d = order c by name PARALLEL 3;
store d into ':OUTPATH:';\,
'sortArgs' => ['-t', ' ', '-k', '1,1'],
},
+ {
+ # Multiple Split + Union + Join
+ 'num' => 9,
+ 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name,
age, gpa);
+b = filter a by gpa >= 3.9;
+b1 = foreach b generate *;
+b2 = foreach b generate *;
+b3 = union onschema b1, b2;
+c = filter a by gpa < 2;
+c1 = foreach c generate *;
+c2 = foreach c generate *;
+c3 = union onschema c1, c2;
+a1 = union onschema b3, c3;
+store a1 into ':OUTPATH:.1';
+d = load ':INPATH:/singlefile/votertab10k' as (name, age, registration,
contributions);
+e = join a1 by name, d by name using 'skewed' PARALLEL 3;
+store e into ':OUTPATH:.2';\,
+ },
] # end of tests
},
Modified:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
---
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld
(original)
+++
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld
Tue Jan 5 08:26:12 2016
@@ -4,22 +4,24 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-79 -> Tez vertex scope-86,Tez vertex scope-88,Tez
vertex scope-97,Tez vertex scope-99,
-Tez vertex scope-86 -> Tez vertex scope-88,
-Tez vertex scope-88 -> Tez vertex scope-103,
-Tez vertex scope-97 -> Tez vertex scope-99,
-Tez vertex scope-99 -> Tez vertex scope-103,
-Tez vertex scope-103 -> Tez vertex scope-108,
-Tez vertex scope-107 -> Tez vertex scope-108,
-Tez vertex scope-108
+Tez vertex scope-77 -> Tez vertex scope-84,Tez vertex scope-86,Tez
vertex scope-95,Tez vertex scope-97,
+Tez vertex scope-84 -> Tez vertex scope-86,
+Tez vertex scope-86 -> Tez vertex scope-101,
+Tez vertex scope-95 -> Tez vertex scope-97,
+Tez vertex scope-97 -> Tez vertex scope-101,
+Tez vertex scope-101 -> Tez vertex scope-118,Tez vertex scope-128,
+Tez vertex scope-118 -> Tez vertex scope-110,Tez vertex scope-128,
+Tez vertex scope-128 -> Tez vertex scope-132,
+Tez vertex scope-110 -> Tez vertex scope-132,
+Tez vertex scope-132
-Tez vertex scope-79
+Tez vertex scope-77
# Plan on vertex
-a: Split - scope-111
+a: Split - scope-141
| |
-| b: Split - scope-109
+| b: Split - scope-139
| | |
-| | POValueOutputTez - scope-90 -> [scope-88]
+| | POValueOutputTez - scope-88 -> [scope-86]
| | |
| | |---b1: New For Each(false,false)[bag] - scope-22
| | | |
@@ -27,7 +29,7 @@ a: Split - scope-111
| | | |
| | | Project[chararray][1] - scope-20
| | |
-| | POValueOutputTez - scope-83 -> [scope-86]
+| | POValueOutputTez - scope-81 -> [scope-84]
| |
| |---b: Filter[bag] - scope-11
| | |
@@ -37,9 +39,9 @@ a: Split - scope-111
| | |
| | |---Constant(2) - scope-13
| |
-| c: Split - scope-110
+| c: Split - scope-140
| | |
-| | POValueOutputTez - scope-101 -> [scope-99]
+| | POValueOutputTez - scope-99 -> [scope-97]
| | |
| | |---c1: New For Each(false,false)[bag] - scope-44
| | | |
@@ -47,7 +49,7 @@ a: Split - scope-111
| | | |
| | | Project[int][0] - scope-42
| | |
-| | POValueOutputTez - scope-94 -> [scope-97]
+| | POValueOutputTez - scope-92 -> [scope-95]
| |
| |---c: Filter[bag] - scope-33
| | |
@@ -68,9 +70,9 @@ a: Split - scope-111
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-86
+Tez vertex scope-84
# Plan on vertex
-POValueOutputTez - scope-91 -> [scope-88]
+POValueOutputTez - scope-89 -> [scope-86]
|
|---b2: New For Each(false,false)[bag] - scope-29
| |
@@ -78,15 +80,15 @@ POValueOutputTez - scope-91 -> [scope-8
| |
| Project[chararray][1] - scope-27
|
- |---POValueInputTez - scope-87 <- scope-79
-Tez vertex scope-88
+ |---POValueInputTez - scope-85 <- scope-77
+Tez vertex scope-86
# Plan on vertex
-POValueOutputTez - scope-105 -> [scope-103]
+POValueOutputTez - scope-103 -> [scope-101]
|
-|---POShuffledValueInputTez - scope-89 <- [scope-79, scope-86]
-Tez vertex scope-97
+|---POShuffledValueInputTez - scope-87 <- [scope-77, scope-84]
+Tez vertex scope-95
# Plan on vertex
-POValueOutputTez - scope-102 -> [scope-99]
+POValueOutputTez - scope-100 -> [scope-97]
|
|---c2: New For Each(false,false)[bag] - scope-51
| |
@@ -94,10 +96,10 @@ POValueOutputTez - scope-102 -> [scope-
| |
| Project[int][0] - scope-49
|
- |---POValueInputTez - scope-98 <- scope-79
-Tez vertex scope-99
+ |---POValueInputTez - scope-96 <- scope-77
+Tez vertex scope-97
# Plan on vertex
-POValueOutputTez - scope-106 -> [scope-103]
+POValueOutputTez - scope-104 -> [scope-101]
|
|---a1: New For Each(false,false)[bag] - scope-57
| |
@@ -105,39 +107,79 @@ POValueOutputTez - scope-106 -> [scope-
| |
| Project[chararray][0] - scope-55
|
- |---POShuffledValueInputTez - scope-100 <- [scope-79, scope-97]
-Tez vertex scope-103
+ |---POShuffledValueInputTez - scope-98 <- [scope-77, scope-95]
+Tez vertex scope-101
# Plan on vertex
-e: Local Rearrange[tuple]{int}(false) - scope-71 -> scope-108
+a1: Split - scope-138
+| |
+| a1: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-62
| |
-| Project[int][0] - scope-72
+| Local Rearrange[tuple]{tuple}(false) - scope-113 -> scope-118
+| | |
+| | Constant(DummyVal) - scope-112
+| |
+| |---New For Each(true,true)[tuple] - scope-117
+| | |
+| | Project[int][0] - scope-73
+| | |
+| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] -
scope-116
+| | |
+| | |---Project[tuple][*] - scope-115
+| |
+| |---PoissonSample - scope-114
+| |
+| |---Local Rearrange[tuple]{int}(false) - scope-111 ->
scope-128
+| | |
+| | Project[int][0] - scope-73
+|
+|---POShuffledValueInputTez - scope-102 <- [scope-86, scope-97]
+Tez vertex scope-118
+# Plan on vertex
+POValueOutputTez - scope-127 -> [scope-110, scope-128]
|
-|---POShuffledValueInputTez - scope-104 <- [scope-88, scope-99]
-Tez vertex scope-107
+|---New For Each(false)[tuple] - scope-126
+ | |
+ |
POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple]
- scope-125
+ | |
+ | |---Project[tuple][*] - scope-124
+ |
+ |---New For Each(false,false)[tuple] - scope-123
+ | |
+ | Constant(-1) - scope-122
+ | |
+ | Project[bag][1] - scope-120
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-119
+Tez vertex scope-128
+# Plan on vertex
+POIdentityInOutTez - scope-129 <- scope-101 -> scope-132
+| |
+| Project[int][0] - scope-73
+Tez vertex scope-110
# Plan on vertex
-e: Local Rearrange[tuple]{int}(false) - scope-73 -> scope-108
+Partition Rearrange[tuple]{int}(false) - scope-130 -> scope-132
| |
| Project[int][0] - scope-74
|
-|---d: New For Each(false,false)[bag] - scope-66
+|---d: New For Each(false,false)[bag] - scope-72
| |
- | Cast[int] - scope-61
+ | Cast[int] - scope-67
| |
- | |---Project[bytearray][0] - scope-60
+ | |---Project[bytearray][0] - scope-66
| |
- | Cast[chararray] - scope-64
+ | Cast[chararray] - scope-70
| |
- | |---Project[bytearray][1] - scope-63
+ | |---Project[bytearray][1] - scope-69
|
- |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-59
-Tez vertex scope-108
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-65
+Tez vertex scope-132
# Plan on vertex
-e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-78
+e: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-76
|
-|---e: New For Each(true,true)[tuple] - scope-77
+|---New For Each(true,true)[tuple] - scope-136
| |
- | Project[bag][1] - scope-75
+ | Project[bag][1] - scope-134
| |
- | Project[bag][2] - scope-76
+ | Project[bag][2] - scope-135
|
- |---e: Package(Packager)[tuple]{int} - scope-70
+ |---Package(Packager)[tuple]{int} - scope-133
Modified:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld
(original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld
Tue Jan 5 08:26:12 2016
@@ -4,36 +4,39 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-107 -> Tez vertex scope-108,
-Tez vertex scope-79 -> Tez vertex scope-108,
-Tez vertex scope-108
+Tez vertex scope-77 -> Tez vertex scope-118,Tez vertex scope-128,
+Tez vertex scope-118 -> Tez vertex scope-110,Tez vertex scope-128,
+Tez vertex scope-128 -> Tez vertex scope-132,
+Tez vertex scope-110 -> Tez vertex scope-132,
+Tez vertex scope-132
-Tez vertex scope-107
+Tez vertex scope-77
# Plan on vertex
-e: Local Rearrange[tuple]{int}(false) - scope-73 -> scope-108
+a: Split - scope-141
| |
-| Project[int][0] - scope-74
-|
-|---d: New For Each(false,false)[bag] - scope-66
- | |
- | Cast[int] - scope-61
- | |
- | |---Project[bytearray][0] - scope-60
- | |
- | Cast[chararray] - scope-64
- | |
- | |---Project[bytearray][1] - scope-63
- |
- |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-59
-Tez vertex scope-79
-# Plan on vertex
-a: Split - scope-111
-| |
-| b: Split - scope-109
+| b: Split - scope-139
| | |
-| | e: Local Rearrange[tuple]{int}(false) - scope-117 ->
scope-108
+| | a1: Split - scope-169
| | | |
-| | | Project[int][0] - scope-118
+| | | a1: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) -
scope-170 -> scope-62
+| | | |
+| | | Local Rearrange[tuple]{tuple}(false) - scope-173 ->
scope-118
+| | | | |
+| | | | Constant(DummyVal) - scope-174
+| | | |
+| | | |---New For Each(true,true)[tuple] - scope-180
+| | | | |
+| | | | Project[int][0] - scope-177
+| | | | |
+| | | |
POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-179
+| | | | |
+| | | | |---Project[tuple][*] - scope-178
+| | | |
+| | | |---PoissonSample - scope-176
+| | | |
+| | | |---Local Rearrange[tuple]{int}(false) - scope-171 ->
scope-128
+| | | | |
+| | | | Project[int][0] - scope-172
| | |
| | |---b1: New For Each(false,false)[bag] - scope-22
| | | |
@@ -41,9 +44,27 @@ a: Split - scope-111
| | | |
| | | Project[chararray][1] - scope-20
| | |
-| | e: Local Rearrange[tuple]{int}(false) - scope-119 ->
scope-108
+| | a1: Split - scope-181
+| | | |
+| | | a1: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) -
scope-182 -> scope-62
+| | | |
+| | | Local Rearrange[tuple]{tuple}(false) - scope-185 ->
scope-118
+| | | | |
+| | | | Constant(DummyVal) - scope-186
| | | |
-| | | Project[int][0] - scope-120
+| | | |---New For Each(true,true)[tuple] - scope-192
+| | | | |
+| | | | Project[int][0] - scope-189
+| | | | |
+| | | |
POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-191
+| | | | |
+| | | | |---Project[tuple][*] - scope-190
+| | | |
+| | | |---PoissonSample - scope-188
+| | | |
+| | | |---Local Rearrange[tuple]{int}(false) - scope-183 ->
scope-128
+| | | | |
+| | | | Project[int][0] - scope-184
| | |
| | |---b2: New For Each(false,false)[bag] - scope-29
| | | |
@@ -59,17 +80,35 @@ a: Split - scope-111
| | |
| | |---Constant(2) - scope-13
| |
-| c: Split - scope-110
+| c: Split - scope-140
| | |
-| | e: Local Rearrange[tuple]{int}(false) - scope-124 ->
scope-108
+| | a1: Split - scope-196
| | | |
-| | | Project[int][0] - scope-125
+| | | a1: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) -
scope-197 -> scope-62
+| | | |
+| | | Local Rearrange[tuple]{tuple}(false) - scope-200 ->
scope-118
+| | | | |
+| | | | Constant(DummyVal) - scope-201
+| | | |
+| | | |---New For Each(true,true)[tuple] - scope-207
+| | | | |
+| | | | Project[int][0] - scope-204
+| | | | |
+| | | |
POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-206
+| | | | |
+| | | | |---Project[tuple][*] - scope-205
+| | | |
+| | | |---PoissonSample - scope-203
+| | | |
+| | | |---Local Rearrange[tuple]{int}(false) - scope-198 ->
scope-128
+| | | | |
+| | | | Project[int][0] - scope-199
| | |
-| | |---a1: New For Each(false,false)[bag] - scope-123
+| | |---a1: New For Each(false,false)[bag] - scope-195
| | | |
-| | | Project[int][1] - scope-121
+| | | Project[int][1] - scope-193
| | | |
-| | | Project[chararray][0] - scope-122
+| | | Project[chararray][0] - scope-194
| | |
| | |---c1: New For Each(false,false)[bag] - scope-44
| | | |
@@ -77,15 +116,33 @@ a: Split - scope-111
| | | |
| | | Project[int][0] - scope-42
| | |
-| | e: Local Rearrange[tuple]{int}(false) - scope-129 ->
scope-108
+| | a1: Split - scope-211
+| | | |
+| | | a1: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) -
scope-212 -> scope-62
+| | | |
+| | | Local Rearrange[tuple]{tuple}(false) - scope-215 ->
scope-118
+| | | | |
+| | | | Constant(DummyVal) - scope-216
| | | |
-| | | Project[int][0] - scope-130
+| | | |---New For Each(true,true)[tuple] - scope-222
+| | | | |
+| | | | Project[int][0] - scope-219
+| | | | |
+| | | |
POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-221
+| | | | |
+| | | | |---Project[tuple][*] - scope-220
+| | | |
+| | | |---PoissonSample - scope-218
+| | | |
+| | | |---Local Rearrange[tuple]{int}(false) - scope-213 ->
scope-128
+| | | | |
+| | | | Project[int][0] - scope-214
| | |
-| | |---a1: New For Each(false,false)[bag] - scope-128
+| | |---a1: New For Each(false,false)[bag] - scope-210
| | | |
-| | | Project[int][1] - scope-126
+| | | Project[int][1] - scope-208
| | | |
-| | | Project[chararray][0] - scope-127
+| | | Project[chararray][0] - scope-209
| | |
| | |---c2: New For Each(false,false)[bag] - scope-51
| | | |
@@ -112,14 +169,53 @@ a: Split - scope-111
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-108
+Tez vertex scope-118
+# Plan on vertex
+POValueOutputTez - scope-127 -> [scope-110, scope-128]
+|
+|---New For Each(false)[tuple] - scope-126
+ | |
+ |
POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple]
- scope-125
+ | |
+ | |---Project[tuple][*] - scope-124
+ |
+ |---New For Each(false,false)[tuple] - scope-123
+ | |
+ | Constant(-1) - scope-122
+ | |
+ | Project[bag][1] - scope-120
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-119
+Tez vertex scope-128
+# Plan on vertex
+POIdentityInOutTez - scope-129 <- scope-77 -> scope-132
+| |
+| Project[int][0] - scope-73
+Tez vertex scope-110
+# Plan on vertex
+Partition Rearrange[tuple]{int}(false) - scope-130 -> scope-132
+| |
+| Project[int][0] - scope-74
+|
+|---d: New For Each(false,false)[bag] - scope-72
+ | |
+ | Cast[int] - scope-67
+ | |
+ | |---Project[bytearray][0] - scope-66
+ | |
+ | Cast[chararray] - scope-70
+ | |
+ | |---Project[bytearray][1] - scope-69
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) -
scope-65
+Tez vertex scope-132
# Plan on vertex
-e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-78
+e: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-76
|
-|---e: New For Each(true,true)[tuple] - scope-77
+|---New For Each(true,true)[tuple] - scope-136
| |
- | Project[bag][1] - scope-75
+ | Project[bag][1] - scope-134
| |
- | Project[bag][2] - scope-76
+ | Project[bag][2] - scope-135
|
- |---e: Package(Packager)[tuple]{int} - scope-70
+ |---Package(Packager)[tuple]{int} - scope-133
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1723016&r1=1723015&r2=1723016&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Tue Jan 5 08:26:12
2016
@@ -732,9 +732,10 @@ public class TestTezCompiler {
"c2 = foreach c generate y, x;" +
"c3 = union c1, c2;" +
"a1 = union onschema b3, c3;" +
+ "store a1 into 'file:///tmp/output1';" +
"d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
- "e = join a1 by x, d by x;" +
- "store e into 'file:///tmp/output';";
+ "e = join a1 by x, d by x using 'skewed';" +
+ "store e into 'file:///tmp/output2';";
setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
run(query,
"test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld");
