Author: daijy
Date: Mon Dec 7 23:32:53 2009
New Revision: 888187
URL: http://svn.apache.org/viewvc?rev=888187&view=rev
Log:
PIG-1127: Logical operator should contains individual copy of schema object
Modified:
hadoop/pig/branches/branch-0.6/CHANGES.txt
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LODistinct.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOLimit.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplit.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOUnion.java
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/branches/branch-0.6/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/CHANGES.txt?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.6/CHANGES.txt Mon Dec 7 23:32:53 2009
@@ -224,6 +224,9 @@
PIG-1128: column pruning causing failure when foreach has user-specified
schema (daijy)
+PIG-1127: Logical operator should contains individual copy of schema object
+(daijy)
+
Release 0.5.0
INCOMPATIBLE CHANGES
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LODistinct.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LODistinct.java
Mon Dec 7 23:32:53 2009
@@ -75,9 +75,17 @@
} else {
fss.add(fs);
mSchema = new Schema(fss);
+ for (int i=0;i<getInput().getSchema().size();i++)
+
mSchema.getField(i).setParent(getInput().getSchema().getField(i).canonicalName,
getInput());
}
} else {
- mSchema = op.getSchema();
+ if (op.getSchema()!=null) {
+ mSchema = new Schema(op.getSchema());
+ for (int i=0;i<op.getSchema().size();i++)
+
mSchema.getField(i).setParent(op.getSchema().getField(i).canonicalName, op);
+ }
+ else
+ mSchema = null;
}
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOFilter.java
Mon Dec 7 23:32:53 2009
@@ -83,7 +83,13 @@
mSchema = new Schema(fss);
}
} else {
- mSchema = input.getSchema();
+ if (getInput().getSchema()!=null) {
+ mSchema = new Schema(input.getSchema());
+ for (int i=0;i<getInput().getSchema().size();i++)
+
mSchema.getField(i).setParent(getInput().getSchema().getField(i).canonicalName,
getInput());
+ }
+ else
+ mSchema = null;
}
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOLimit.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOLimit.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOLimit.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOLimit.java
Mon Dec 7 23:32:53 2009
@@ -63,7 +63,13 @@
public Schema getSchema() throws FrontendException {
if (!mIsSchemaComputed) {
try {
- mSchema = getInput().getSchema();
+ if (getInput().getSchema()!=null) {
+ mSchema = new Schema(getInput().getSchema());
+ for (int i=0;i<getInput().getSchema().size();i++)
+
mSchema.getField(i).setParent(getInput().getSchema().getField(i).canonicalName,
getInput());
+ }
+ else
+ mSchema = null;
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
mSchema = null;
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSort.java
Mon Dec 7 23:32:53 2009
@@ -155,7 +155,13 @@
mSchema = new Schema(fss);
}
} else {
- mSchema = op.getSchema();
+ if (getInput().getSchema()!=null) {
+ mSchema = new Schema(op.getSchema());
+ for (int i=0;i<getInput().getSchema().size();i++)
+
mSchema.getField(i).setParent(getInput().getSchema().getField(i).canonicalName,
getInput());
+ }
+ else
+ mSchema = null;
}
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplit.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplit.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplit.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplit.java
Mon Dec 7 23:32:53 2009
@@ -86,7 +86,14 @@
String msg = "Could not find operator in plan";
throw new FrontendException(msg, errCode,
PigException.INPUT, false, null);
}
- mSchema = s.iterator().next().getSchema();
+ LogicalOperator input = s.iterator().next();
+ if (input.getSchema()!=null) {
+ mSchema = new Schema(input.getSchema());
+ for (int i=0;i<input.getSchema().size();i++)
+
mSchema.getField(i).setParent(input.getSchema().getField(i).canonicalName,
input);
+ }
+ else
+ mSchema = null;
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
mSchema = null;
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
Mon Dec 7 23:32:53 2009
@@ -89,7 +89,13 @@
String msg = "Could not find operator in plan";
throw new FrontendException(msg, errCode,
PigException.INPUT, false, null);
}
- mSchema = input.getSchema();
+ if (input.getSchema()!=null) {
+ mSchema = new Schema(input.getSchema());
+ for (int i=0;i<input.getSchema().size();i++)
+
mSchema.getField(i).setParent(input.getSchema().getField(i).canonicalName,
input);
+ }
+ else
+ mSchema = null;
mIsSchemaComputed = true;
} catch (FrontendException fe) {
mSchema = null;
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOUnion.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOUnion.java
Mon Dec 7 23:32:53 2009
@@ -71,7 +71,10 @@
String msg = "Could not find operator in plan";
throw new FrontendException(msg, errCode,
PigException.INPUT, false, null);
}
- mSchema = op.getSchema();
+ if (op.getSchema()!=null)
+ mSchema = new Schema(op.getSchema());
+ else
+ mSchema = null;
while(iter.hasNext()) {
op = iter.next();
if(null != mSchema) {
Modified:
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
(original)
+++
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
Mon Dec 7 23:32:53 2009
@@ -3126,14 +3126,6 @@
HandleSpec streamOutputSpec = command.getOutputSpec();
FuncSpec streamLoaderSpec = new
FuncSpec(streamOutputSpec.getSpec());
return streamLoaderSpec;
- } else if ((op instanceof LOFilter)
- || (op instanceof LODistinct)
- || (op instanceof LOSort)
- || (op instanceof LOSplit)
- || (op instanceof LOSplitOutput)
- || (op instanceof LOLimit)) {
- LogicalPlan lp = op.getPlan();
- return getLoadFuncSpec(lp.getPredecessors(op).get(0),
parentCanonicalName);
}
Schema s = op.getSchema();
Modified:
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java?rev=888187&r1=888186&r2=888187&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
(original)
+++
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Mon Dec 7 23:32:53 2009
@@ -53,6 +53,7 @@
File tmpFile7;
File tmpFile8;
File tmpFile9;
+ File tmpFile10;
File logFile;
Logger logger;
@@ -139,6 +140,10 @@
ps.println("2\t[key1#2,key2#4]\t[key3#8,key4#9]");
ps.close();
+ tmpFile10 = File.createTempFile("prune", "txt");
+ ps = new PrintStream(new FileOutputStream(tmpFile10));
+ ps.println("1\t[1#1,2#1]\t2");
+ ps.close();
}
@@ -1474,4 +1479,23 @@
assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A:
$1",
"No map keys pruned for A"}));
}
+
+ // See PIG-1127
+ @Test
+ public void testSharedSchemaObject() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile10.toString()) + "' AS (a0, a1:map[], a2);");
+ pigServer.registerQuery("B = foreach A generate a1;");
+ pigServer.registerQuery("C = limit B 10;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("([2#1,1#1])"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0,
$2",
+ "No map keys pruned for A"}));
+ }
}
