Author: daijy
Date: Fri Dec 11 18:42:29 2009
New Revision: 889758
URL: http://svn.apache.org/viewvc?rev=889758&view=rev
Log:
PIG-1142: Got NullPointerException merge join with pruning
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java?rev=889758&r1=889757&r2=889758&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
Fri Dec 11 18:42:29 2009
@@ -817,7 +817,8 @@
errCode, PigException.BUG);
}
- for (Pair<Integer, Integer> column : columns) {
+ for (int i=columns.size()-1;i>=0;i--) {
+ Pair<Integer, Integer> column = columns.get(i);
if (column.first < 0 || column.first > predecessors.size()) {
int errCode = 2191;
throw new FrontendException("No input " + column.first
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=889758&r1=889757&r2=889758&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
Fri Dec 11 18:42:29 2009
@@ -268,7 +268,8 @@
.warn("Cannot prune columns in filter, no schema
information found");
return false;
}
- for (Pair<Integer, Integer> column : columns) {
+ for (int i=columns.size()-1;i>=0;i--) {
+ Pair<Integer, Integer> column = columns.get(i);
if (column.first != 0) {
int errCode = 2191;
throw new FrontendException(
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=889758&r1=889757&r2=889758&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
Fri Dec 11 18:42:29 2009
@@ -385,7 +385,8 @@
if (predecessors == null)
return false;
- for (Pair<Integer, Integer> column : columns) {
+ for (int i=columns.size()-1;i>=0;i--) {
+ Pair<Integer, Integer> column = columns.get(i);
if (column.first != 0) {
int errCode = 2191;
throw new FrontendException(
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=889758&r1=889757&r2=889758&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
Fri Dec 11 18:42:29 2009
@@ -283,7 +283,8 @@
return false;
}
- for (Pair<Integer, Integer> column : columns) {
+ for (int i=columns.size()-1;i>=0;i--) {
+ Pair<Integer, Integer> column = columns.get(i);
if (column.first != 0) {
int errCode = 2191;
throw new FrontendException(
Modified:
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java?rev=889758&r1=889757&r2=889758&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
(original)
+++
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Fri Dec 11 18:42:29 2009
@@ -1562,5 +1562,94 @@
assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
"No map keys pruned for A", "Columns pruned for B: $0, $1",
"No map keys pruned for B"}));
- }
+ }
+
+ @Test
+ public void testFilter4() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2:int);");
+ pigServer.registerQuery("B = filter A by a2==3;");
+ pigServer.registerQuery("C = foreach B generate $2;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(3)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0,
$1",
+ "No map keys pruned for A"}));
+ }
+
+ @Test
+ public void testSplit3() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2:int);");
+ pigServer.registerQuery("split A into B if a2==3, C if a2<3;");
+ pigServer.registerQuery("C = foreach B generate $2;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(3)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0,
$1",
+ "No map keys pruned for A"}));
+ }
+
+ @Test
+ public void testOrderBy3() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = order A by a2;");
+ pigServer.registerQuery("C = foreach B generate a2;");
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+
+ assertTrue(t.size()==1);
+ assertTrue(t.toString().equals("(2)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+
+ assertTrue(t.size()==1);
+ assertTrue(t.toString().equals("(3)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0,
$1",
+ "No map keys pruned for A"}));
+ }
+
+ @Test
+ public void testCogroup9() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS (b0, b1, b2);");
+ pigServer.registerQuery("C = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS (c0, c1, c2);");
+ pigServer.registerQuery("D = cogroup A by a2, B by b2, C by c2;");
+ pigServer.registerQuery("E = foreach D generate $1, $2;");
+ Iterator<Tuple> iter = pigServer.openIterator("E");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+
+ assertTrue(t.size()==2);
+ assertTrue(t.toString().equals("({(2,5,2)},{(2,5,2)})"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+
+ assertTrue(t.size()==2);
+ assertTrue(t.toString().equals("({(1,2,3)},{(1,2,3)})"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for C: $0,
$1",
+ "No map keys pruned for C"}));
+ }
+
}
