Author: daijy
Date: Mon Dec 7 21:55:41 2009
New Revision: 888153
URL: http://svn.apache.org/viewvc?rev=888153&view=rev
Log:
PIG-1128: column pruning causing failure when foreach has user-specified schema
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=888153&r1=888152&r2=888153&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Dec 7 21:55:41 2009
@@ -45,8 +45,6 @@
OPTIMIZATIONS
-PIG-922: Logical optimizer: push up project (daijy)
-
BUG FIXES
PIG-1105: COUNT_STAR accumulate interface implementation cases failure
@@ -85,6 +83,8 @@
INCOMPATIBLE CHANGES
+PIG-922: Logical optimizer: push up project (daijy)
+
IMPROVEMENTS
PIG-1084: Pig 0.6.0 Documentation improvements (chandec via olgan)
@@ -182,6 +182,8 @@
OPTIMIZATIONS
+PIG-922: Logical optimizer: push up project (daijy)
+
BUG FIXES
PIG-1080: PigStorage may miss records when loading a file (rding via olgan)
@@ -251,6 +253,9 @@
PIG-1060: MultiQuery optimization throws error for multi-level splits (rding
via daijy)
+PIG-1128: column pruning causing failure when foreach has user-specified
+schema (daijy)
+
Release 0.5.0
INCOMPATIBLE CHANGES
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=888153&r1=888152&r2=888153&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java Mon
Dec 7 21:55:41 2009
@@ -944,6 +944,12 @@
}
while (planToRemove.size() > 0) {
int index = planToRemove.get(planToRemove.size()-1);
+ if (mUserDefinedSchema!=null) {
+ for (int i=mUserDefinedSchema.size()-1;i>=0;i--) {
+ if (getRelevantPlan(0, i)==mForEachPlans.get(index))
+ mUserDefinedSchema.remove(i);
+ }
+ }
mForEachPlans.remove(index);
mFlatten.remove(index);
planToRemove.remove(planToRemove.size()-1);
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=888153&r1=888152&r2=888153&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Mon Dec 7
21:55:41 2009
@@ -1453,5 +1453,28 @@
assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
"No map keys pruned for A", "Columns pruned for B: $0",
"No map keys pruned for B"}));
- }
+ }
+
+ // See PIG-1128
+ @Test
+ public void testUserDefinedSchema() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString()) + "' AS ( c1 : chararray, c2 : int);");
+ pigServer.registerQuery("B = foreach A generate c1 as c1 : chararray,
c2 as c2 : int, 'CA' as state : chararray;");
+ pigServer.registerQuery("C = foreach B generate c1 as c1 :
chararray;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(2)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A:
$1",
+ "No map keys pruned for A"}));
+ }
}
