Author: daijy
Date: Fri Aug 14 16:27:49 2009
New Revision: 804282
URL: http://svn.apache.org/viewvc?rev=804282&view=rev
Log:
PIG-913: Error in Pig script when grouping on chararray column
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=804282&r1=804281&r2=804282&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Aug 14 16:27:49 2009
@@ -49,6 +49,8 @@
BUG FIXES
+ PIG-913: Error in Pig script when grouping on chararray column (daijy)
+
PIG-907: Provide multiple version of HashFNV (Piggybank) (daijy)
PIG-905: TOKENIZE throws exception on null data (daijy)
Modified:
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java?rev=804282&r1=804281&r2=804282&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java
Fri Aug 14 16:27:49 2009
@@ -206,73 +206,6 @@
insertBetween(after, newNode, successors.get(0), projectionMapping);
}
- /**
- * Remove a node in the middle of a linear chain. This includes removing
the node
- * from the plan and reconnect the nodes before and after the node as well
as rebuilding
- * all of the schemas. This node should have one and only one predecessor
and zero or one
- * successor
- * @param nodeToRemove Node to remove
- * @param projectionMapping A map that defines how projections in after
- * relate to projections in nodeToRemove. Keys are the projection offsets
in
- * after, values are the new offsets in nodeToRemove. If this field is
null,
- * then it will be assumed that the mapping is 1-1.
- * @throws VisitorException, FrontendException
- */
- protected void removeFromChain(
- LogicalOperator nodeToRemove,
- Map<Integer, Integer> projectionMapping)
- throws VisitorException, FrontendException {
-
- List<LogicalOperator> afterNodes = mPlan.getPredecessors(nodeToRemove);
- if (afterNodes.size()!=1)
- throw new RuntimeException("removeFromChain only valid to
remove " +
- "node has one predecessor.");
- List<LogicalOperator> beforeNodes = mPlan.getSuccessors(nodeToRemove);
- if (beforeNodes!=null && beforeNodes.size()!=1)
- throw new RuntimeException("removeFromChain only valid to
remove " +
- "node has one successor.");
-
- // Get after and before node
- LogicalOperator after = mPlan.getPredecessors(nodeToRemove).get(0);
- LogicalOperator before = null;
- if (beforeNodes!=null)
- before = mPlan.getSuccessors(nodeToRemove).get(0);
-
- // Remove nodeToRemove from plan
- mPlan.remove(nodeToRemove);
- if (before!=null)
- {
- // Shortcut nodeToRemove.
- mPlan.connect(after, before);
-
- // Visit all the inner plans of before and change their
projects to
- // connect to after instead of nodeToRemove.
- // Find right inner plan(s) to visit
- List<LogicalPlan> plans = new ArrayList<LogicalPlan>();
- if (before instanceof LOCogroup) {
-
plans.addAll((((LOCogroup)before).getGroupByPlans()).values());
- } else if (before instanceof LOSort) {
- plans.addAll(((LOSort)before).getSortColPlans());
- } else if (before instanceof LOFilter) {
- plans.add(((LOFilter)before).getComparisonPlan());
- } else if (before instanceof LOSplitOutput) {
- plans.add(((LOSplitOutput)before).getConditionPlan());
- } else if (before instanceof LOForEach) {
- plans.addAll(((LOForEach)before).getForEachPlans());
- }
-
- for (LogicalPlan lp : plans) {
- ProjectFixerUpper pfu =
- new ProjectFixerUpper(lp, nodeToRemove, after,
projectionMapping);
- pfu.visit();
- }
-
- }
-
- // Now rebuild the schemas
- // rebuildSchemas();
- }
-
@Override
public void reset() {
// do nothing
Modified:
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java?rev=804282&r1=804281&r2=804282&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
Fri Aug 14 16:27:49 2009
@@ -159,7 +159,7 @@
if (prepredecessor!=null)
{
try {
- removeFromChain(limit, null);
+ mPlan.removeAndReconnect(limit);
insertBetween(prepredecessor,
limit, predecessor, null);
} catch (Exception e) {
@@ -213,7 +213,7 @@
else
sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
try {
- removeFromChain(limit, null);
+ mPlan.removeAndReconnect(limit);
} catch (Exception e) {
int errCode = 2012;
String msg = "Can not remove LOLimit after LOSort";
@@ -227,7 +227,7 @@
LOLimit beforeLimit = (LOLimit)predecessor;
beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
try {
- removeFromChain(limit, null);
+ mPlan.removeAndReconnect(limit);
} catch (Exception e) {
int errCode = 2012;
String msg = "Can not remove LOLimit after LOLimit";
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java?rev=804282&r1=804281&r2=804282&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java Fri Aug
14 16:27:49 2009
@@ -185,6 +185,17 @@
}
@Test
+ //See bug PIG-913
+ public void testOPLimit10Optimizer() throws Exception {
+ planTester.buildPlan("A = load 'myfile' AS (s:chararray);");
+ planTester.buildPlan("B = limit A 100;");
+ LogicalPlan plan = planTester.buildPlan("C = GROUP B by $0;");
+ optimizePlan(plan);
+ compareWithGoldenFile(plan, FILE_BASE_LOCATION + "optlimitplan10.dot");
+ }
+
+
+ @Test
//Test to ensure that the right exception is thrown
public void testErrImplicitSplitInserter() throws Exception {
LogicalPlan lp = new LogicalPlan();
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot?rev=804282&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot
(added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot
Fri Aug 14 16:27:49 2009
@@ -0,0 +1,7 @@
+digraph graph1 {
+ LOLoad1061 -> LOLimit1062;
+ LOLimit1062 [limit="100"];
+ LOLimit1062 -> LOForEach1067;
+ LOForEach1067 [hasflat="false"];
+ LOForEach1067 -> LOCogroup1064;
+}
