Author: daijy Date: Fri Dec 11 18:42:29 2009 New Revision: 889758 URL: http://svn.apache.org/viewvc?rev=889758&view=rev Log: PIG-1142: Got NullPointerException merge join with pruning
Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java?rev=889758&r1=889757&r2=889758&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java (original) +++ hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java Fri Dec 11 18:42:29 2009 @@ -817,7 +817,8 @@ errCode, PigException.BUG); } - for (Pair<Integer, Integer> column : columns) { + for (int i=columns.size()-1;i>=0;i--) { + Pair<Integer, Integer> column = columns.get(i); if (column.first < 0 || column.first > predecessors.size()) { int errCode = 2191; throw new FrontendException("No input " + column.first Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=889758&r1=889757&r2=889758&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java (original) +++ hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java Fri Dec 11 18:42:29 2009 @@ -268,7 +268,8 @@ .warn("Cannot prune columns in filter, no schema information found"); return false; } - for (Pair<Integer, Integer> column : columns) { + for (int i=columns.size()-1;i>=0;i--) { + Pair<Integer, Integer> column = columns.get(i); if (column.first != 0) { int errCode = 2191; throw new FrontendException( Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=889758&r1=889757&r2=889758&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java (original) +++ hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java Fri Dec 11 18:42:29 2009 @@ -385,7 +385,8 @@ if (predecessors == null) return false; - for (Pair<Integer, Integer> column : columns) { + for (int i=columns.size()-1;i>=0;i--) { + Pair<Integer, Integer> column = columns.get(i); if (column.first != 0) { int errCode = 2191; throw new FrontendException( Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=889758&r1=889757&r2=889758&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java (original) +++ hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java Fri Dec 11 18:42:29 2009 @@ -283,7 +283,8 @@ return false; } - for (Pair<Integer, Integer> column : columns) { + for (int i=columns.size()-1;i>=0;i--) { + Pair<Integer, Integer> column = columns.get(i); if (column.first != 0) { int errCode = 2191; throw new FrontendException( Modified: hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java?rev=889758&r1=889757&r2=889758&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java (original) +++ hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java Fri Dec 11 18:42:29 2009 @@ -1562,5 +1562,94 @@ assertTrue(checkLogFileMessage(new String[]{"No column pruned for A", "No map keys pruned for A", "Columns pruned for B: $0, $1", "No map keys pruned for B"})); - } + } + + @Test + public void testFilter4() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2:int);"); + pigServer.registerQuery("B = filter A by a2==3;"); + pigServer.registerQuery("C = foreach B generate $2;"); + + Iterator<Tuple> iter = pigServer.openIterator("C"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(3)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0, $1", + "No map keys pruned for A"})); + } + + @Test + public void testSplit3() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2:int);"); + pigServer.registerQuery("split A into B if a2==3, C if a2<3;"); + pigServer.registerQuery("C = foreach B generate $2;"); + + Iterator<Tuple> iter = pigServer.openIterator("C"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(3)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0, $1", + "No map keys pruned for A"})); + } + + @Test + public void testOrderBy3() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2);"); + pigServer.registerQuery("B = order A by a2;"); + pigServer.registerQuery("C = foreach B generate a2;"); + Iterator<Tuple> iter = pigServer.openIterator("C"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + + assertTrue(t.size()==1); + assertTrue(t.toString().equals("(2)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + + assertTrue(t.size()==1); + assertTrue(t.toString().equals("(3)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0, $1", + "No map keys pruned for A"})); + } + + @Test + public void testCogroup9() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2);"); + pigServer.registerQuery("B = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS (b0, b1, b2);"); + pigServer.registerQuery("C = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS (c0, c1, c2);"); + pigServer.registerQuery("D = cogroup A by a2, B by b2, C by c2;"); + pigServer.registerQuery("E = foreach D generate $1, $2;"); + Iterator<Tuple> iter = pigServer.openIterator("E"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + + assertTrue(t.size()==2); + assertTrue(t.toString().equals("({(2,5,2)},{(2,5,2)})")); + + assertTrue(iter.hasNext()); + t = iter.next(); + + assertTrue(t.size()==2); + assertTrue(t.toString().equals("({(1,2,3)},{(1,2,3)})")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for C: $0, $1", + "No map keys pruned for C"})); + } + }