Author: daijy
Date: Mon Dec 7 22:02:18 2009
New Revision: 888157
URL: http://svn.apache.org/viewvc?rev=888157&view=rev
Log:
PIG-1128: column pruning causing failure when foreach has user-specified schema
Modified:
hadoop/pig/branches/branch-0.6/CHANGES.txt
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/branches/branch-0.6/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/CHANGES.txt?rev=888157&r1=888156&r2=888157&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.6/CHANGES.txt Mon Dec 7 22:02:18 2009
@@ -22,6 +22,8 @@
INCOMPATIBLE CHANGES
+PIG-922: Logical optimizer: push up project (daijy)
+
IMPROVEMENTS
PIG-1129: Pig UDF doc: fieldsToRead function (chandec via olgan)
@@ -219,6 +221,9 @@
PIG-1060: MultiQuery optimization throws error for multi-level splits (rding
via daijy)
+PIG-1128: column pruning causing failure when foreach has user-specified
+schema (daijy)
+
Release 0.5.0
INCOMPATIBLE CHANGES
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=888157&r1=888156&r2=888157&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
Mon Dec 7 22:02:18 2009
@@ -944,6 +944,12 @@
}
while (planToRemove.size() > 0) {
int index = planToRemove.get(planToRemove.size()-1);
+ if (mUserDefinedSchema!=null) {
+ for (int i=mUserDefinedSchema.size()-1;i>=0;i--) {
+ if (getRelevantPlan(0, i)==mForEachPlans.get(index))
+ mUserDefinedSchema.remove(i);
+ }
+ }
mForEachPlans.remove(index);
mFlatten.remove(index);
planToRemove.remove(planToRemove.size()-1);
Modified:
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java?rev=888157&r1=888156&r2=888157&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
(original)
+++
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Mon Dec 7 22:02:18 2009
@@ -1450,5 +1450,28 @@
assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
"No map keys pruned for A", "Columns pruned for B: $0",
"No map keys pruned for B"}));
- }
+ }
+
+ // See PIG-1128
+ @Test
+ public void testUserDefinedSchema() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS ( c1 : chararray, c2 : int);");
+ pigServer.registerQuery("B = foreach A generate c1 as c1 : chararray,
c2 as c2 : int, 'CA' as state : chararray;");
+ pigServer.registerQuery("C = foreach B generate c1 as c1 :
chararray;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(2)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A:
$1",
+ "No map keys pruned for A"}));
+ }
}
