Author: daijy Date: Fri Aug 14 16:27:49 2009 New Revision: 804282 URL: http://svn.apache.org/viewvc?rev=804282&view=rev Log: PIG-913: Error in Pig script when grouping on chararray column
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=804282&r1=804281&r2=804282&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Aug 14 16:27:49 2009 @@ -49,6 +49,8 @@ BUG FIXES + PIG-913: Error in Pig script when grouping on chararray column (daijy) + PIG-907: Provide multiple version of HashFNV (Piggybank) (daijy) PIG-905: TOKENIZE throws exception on null data (daijy) Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java?rev=804282&r1=804281&r2=804282&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java Fri Aug 14 16:27:49 2009 @@ -206,73 +206,6 @@ insertBetween(after, newNode, successors.get(0), projectionMapping); } - /** - * Remove a node in the middle of a linear chain. This includes removing the node - * from the plan and reconnect the nodes before and after the node as well as rebuilding - * all of the schemas. This node should have one and only one predecessor and zero or one - * successor - * @param nodeToRemove Node to remove - * @param projectionMapping A map that defines how projections in after - * relate to projections in nodeToRemove. Keys are the projection offsets in - * after, values are the new offsets in nodeToRemove. If this field is null, - * then it will be assumed that the mapping is 1-1. - * @throws VisitorException, FrontendException - */ - protected void removeFromChain( - LogicalOperator nodeToRemove, - Map<Integer, Integer> projectionMapping) - throws VisitorException, FrontendException { - - List<LogicalOperator> afterNodes = mPlan.getPredecessors(nodeToRemove); - if (afterNodes.size()!=1) - throw new RuntimeException("removeFromChain only valid to remove " + - "node has one predecessor."); - List<LogicalOperator> beforeNodes = mPlan.getSuccessors(nodeToRemove); - if (beforeNodes!=null && beforeNodes.size()!=1) - throw new RuntimeException("removeFromChain only valid to remove " + - "node has one successor."); - - // Get after and before node - LogicalOperator after = mPlan.getPredecessors(nodeToRemove).get(0); - LogicalOperator before = null; - if (beforeNodes!=null) - before = mPlan.getSuccessors(nodeToRemove).get(0); - - // Remove nodeToRemove from plan - mPlan.remove(nodeToRemove); - if (before!=null) - { - // Shortcut nodeToRemove. - mPlan.connect(after, before); - - // Visit all the inner plans of before and change their projects to - // connect to after instead of nodeToRemove. - // Find right inner plan(s) to visit - List<LogicalPlan> plans = new ArrayList<LogicalPlan>(); - if (before instanceof LOCogroup) { - plans.addAll((((LOCogroup)before).getGroupByPlans()).values()); - } else if (before instanceof LOSort) { - plans.addAll(((LOSort)before).getSortColPlans()); - } else if (before instanceof LOFilter) { - plans.add(((LOFilter)before).getComparisonPlan()); - } else if (before instanceof LOSplitOutput) { - plans.add(((LOSplitOutput)before).getConditionPlan()); - } else if (before instanceof LOForEach) { - plans.addAll(((LOForEach)before).getForEachPlans()); - } - - for (LogicalPlan lp : plans) { - ProjectFixerUpper pfu = - new ProjectFixerUpper(lp, nodeToRemove, after, projectionMapping); - pfu.visit(); - } - - } - - // Now rebuild the schemas - // rebuildSchemas(); - } - @Override public void reset() { // do nothing Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java?rev=804282&r1=804281&r2=804282&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java Fri Aug 14 16:27:49 2009 @@ -159,7 +159,7 @@ if (prepredecessor!=null) { try { - removeFromChain(limit, null); + mPlan.removeAndReconnect(limit); insertBetween(prepredecessor, limit, predecessor, null); } catch (Exception e) { @@ -213,7 +213,7 @@ else sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit()); try { - removeFromChain(limit, null); + mPlan.removeAndReconnect(limit); } catch (Exception e) { int errCode = 2012; String msg = "Can not remove LOLimit after LOSort"; @@ -227,7 +227,7 @@ LOLimit beforeLimit = (LOLimit)predecessor; beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit()); try { - removeFromChain(limit, null); + mPlan.removeAndReconnect(limit); } catch (Exception e) { int errCode = 2012; String msg = "Can not remove LOLimit after LOLimit"; Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java?rev=804282&r1=804281&r2=804282&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalOptimizer.java Fri Aug 14 16:27:49 2009 @@ -185,6 +185,17 @@ } @Test + //See bug PIG-913 + public void testOPLimit10Optimizer() throws Exception { + planTester.buildPlan("A = load 'myfile' AS (s:chararray);"); + planTester.buildPlan("B = limit A 100;"); + LogicalPlan plan = planTester.buildPlan("C = GROUP B by $0;"); + optimizePlan(plan); + compareWithGoldenFile(plan, FILE_BASE_LOCATION + "optlimitplan10.dot"); + } + + + @Test //Test to ensure that the right exception is thrown public void testErrImplicitSplitInserter() throws Exception { LogicalPlan lp = new LogicalPlan(); Added: hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot?rev=804282&view=auto ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot (added) +++ hadoop/pig/trunk/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot Fri Aug 14 16:27:49 2009 @@ -0,0 +1,7 @@ +digraph graph1 { + LOLoad1061 -> LOLimit1062; + LOLimit1062 [limit="100"]; + LOLimit1062 -> LOForEach1067; + LOForEach1067 [hasflat="false"]; + LOForEach1067 -> LOCogroup1064; +}