Author: gates
Date: Tue Jun 17 17:39:00 2008
New Revision: 668885
URL: http://svn.apache.org/viewvc?rev=668885&view=rev
Log:
PIG-158 Santhosh's cast_fix patch.
Modified:
incubator/pig/branches/types/build.xml
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Modified: incubator/pig/branches/types/build.xml
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Tue Jun 17 17:39:00 2008
@@ -234,11 +234,7 @@
<exclude name="**/TestCompressedFiles.java" />
<exclude name="**/TestEvalPipeline.java" />
<exclude name="**/TestFilterOpNumeric.java" />
- <exclude name="**/TestFilterOpString.java" />
- <exclude name="**/TestInfixArithmetic.java" />
- <!-- <exclude name="**/TestMapReduce.java" /> -->
<exclude name="**/TestPigFile.java" />
- <exclude name="**/TestPigSplit.java" />
<exclude name="**/TestStoreOld.java" />
<!-- Excluced because we don't want to run them -->
<exclude name="**/TypeCheckingTestUtil.java" />
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
Tue Jun 17 17:39:00 2008
@@ -19,6 +19,7 @@
import java.util.Collection;
import java.util.List;
+import java.util.ArrayList;
import java.io.IOException;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -60,13 +61,18 @@
if (!mIsSchemaComputed && (null == mSchema)) {
// Get the schema of the parent
Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+ ArrayList<Schema.FieldSchema> fss = new
ArrayList<Schema.FieldSchema>();
try {
LogicalOperator op = s.iterator().next();
if (null == op) {
- log.debug("getSchema: Operator not in plan");
throw new FrontendException("Could not find operator in
plan");
}
- mSchema = s.iterator().next().getSchema();
+ if(op instanceof ExpressionOperator) {
+ fss.add(((ExpressionOperator)op).getFieldSchema());
+ mSchema = new Schema(fss);
+ } else {
+ mSchema = op.getSchema();
+ }
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
mSchema = null;
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
Tue Jun 17 17:39:00 2008
@@ -18,6 +18,7 @@
package org.apache.pig.impl.logicalLayer;
import java.util.List;
+import java.util.ArrayList;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.OperatorKey;
@@ -64,8 +65,14 @@
@Override
public Schema getSchema() throws FrontendException {
if (!mIsSchemaComputed && (null == mSchema)) {
+ ArrayList<Schema.FieldSchema> fss = new
ArrayList<Schema.FieldSchema>();
try {
- mSchema = mInput.getSchema();
+ if(mInput instanceof ExpressionOperator) {
+ fss.add(((ExpressionOperator)mInput).getFieldSchema());
+ mSchema = new Schema(fss);
+ } else {
+ mSchema = mInput.getSchema();
+ }
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
mSchema = null;
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java
Tue Jun 17 17:39:00 2008
@@ -21,11 +21,17 @@
import java.util.Iterator;
import java.util.List;
import java.io.PrintStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.Collection;
+import java.util.Collections;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.DepthFirstWalker;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.MultiMap;
import org.apache.pig.impl.plan.VisitorException;
@@ -35,299 +41,163 @@
public class LOPrinter extends LOVisitor {
private PrintStream mStream = null;
- private int mIndent = 0;
+ private String TAB1 = " ";
+ private String TABMore = "| ";
+ private String LSep = "|\n|---";
+ private String USep = "| |\n| ";
+ private int levelCntr = -1;
+ private OutputStream printer;
/**
* @param ps PrintStream to output plan information to
* @param plan Logical plan to print
*/
public LOPrinter(PrintStream ps, LogicalPlan plan) {
+ //super(plan, new DependencyOrderWalker(plan));
super(plan, new DepthFirstWalker(plan));
+ mStream = ps;
}
- public void visit(LOAdd a) throws VisitorException {
- visitBinary(a, "+");
- }
-
- public void visit(LOAnd a) throws VisitorException {
- visitBinary(a, "AND");
- }
-
- public void visit(LOBinCond bc) throws VisitorException {
- print(bc);
- mStream.print(" COND: (");
- bc.getCond().visit(this);
- mStream.print(") TRUE: (");
- bc.getLhsOp().visit(this);
- mStream.print(") FALSE (");
- bc.getRhsOp().visit(this);
- mStream.print(")");
- }
-
- public void visit(LOCogroup g) throws VisitorException {
- print(g);
- mStream.print("GROUP BY PLANS:");
- MultiMap<LogicalOperator, LogicalPlan> plans = g.getGroupByPlans();
- for (LogicalOperator lo : plans.keySet()) {
- // Visit the associated plans
- for (LogicalPlan plan : plans.get(lo)) {
- mIndent++;
- pushWalker(new DepthFirstWalker(plan));
- visit();
- popWalker();
- mIndent--;
- }
- mStream.println();
- }
- // Visit input operators
- for (LogicalOperator lo : plans.keySet()) {
- // Visit the operator
- lo.visit(this);
+ @Override
+ public void visit() throws VisitorException {
+ try {
+ mStream.write(depthFirstLP().getBytes());
+ } catch (IOException e) {
+ throw new VisitorException(e.getMessage());
}
}
-
- public void visit(LOConst c) throws VisitorException {
- print(c);
- mStream.print(" VALUE (" + c.getValue() + ")");
- }
- public void visit(LOCross c) throws VisitorException {
- print(c);
- mStream.println();
- super.visit(c);
+ public void print(OutputStream printer) throws VisitorException,
IOException {
+ this.printer = printer;
+ printer.write(depthFirstLP().getBytes());
}
- public void visit(LODistinct d) throws VisitorException {
- print(d);
- mStream.println();
- super.visit(d);
- }
-
- public void visit(LODivide d) throws VisitorException {
- visitBinary(d, "/");
- }
- public void visit(LOEqual e) throws VisitorException {
- visitBinary(e, "==");
- }
-
- public void visit(LOFilter f) throws VisitorException {
- print(f);
- mStream.print(" COMP: ");
- mIndent++;
- pushWalker(new DepthFirstWalker(f.getComparisonPlan()));
- visit();
- mIndent--;
- mStream.println();
- f.getInput().visit(this);
- }
-
- public void visit(LOForEach f) throws VisitorException {
- print(f);
- mStream.print(" PLAN: ");
- mIndent++;
- pushWalker(new DepthFirstWalker(f.getForEachPlan()));
- visit();
- mIndent--;
- mStream.println();
- // Visit our input
- mPlan.getPredecessors((LogicalOperator)f).get(0).visit(this);
- }
-
- public void visit(LOGreaterThan gt) throws VisitorException {
- visitBinary(gt, ">");
- }
-
- public void visit(LOGreaterThanEqual gte) throws VisitorException {
- visitBinary(gte, ">=");
- }
-
- public void visit(LOLesserThan lt) throws VisitorException {
- visitBinary(lt, "<");
- }
-
- public void visit(LOLesserThanEqual lte) throws VisitorException {
- visitBinary(lte, "<=");
- }
-
- public void visit(LOLoad load) throws VisitorException {
- print(load);
- mStream.print(" FILE: " + load.getInputFile().getFileName());
- mStream.print(" FUNC: " + load.getLoadFunc().getClass().getName());
- mStream.println();
- }
-
- public void visit(LOMapLookup mlu) throws VisitorException {
- print(mlu);
- mStream.print("(");
- mlu.getMap().visit(this);
- mStream.print(")# " + mlu.getOperatorKey());
- }
-
- public void visit(LOMod m) throws VisitorException {
- visitBinary(m, "MOD");
- }
-
- public void visit(LOMultiply m) throws VisitorException {
- visitBinary(m, "*");
- }
-
- public void visit(LONegative n) throws VisitorException {
- visitUnary(n, "-");
- }
-
- public void visit(LONot n) throws VisitorException {
- visitUnary(n, "NOT");
+ protected String depthFirstLP() throws VisitorException {
+ StringBuilder sb = new StringBuilder();
+ List<LogicalOperator> leaves = mPlan.getLeaves();
+ Collections.sort(leaves);
+ for (LogicalOperator leaf : leaves) {
+ sb.append(depthFirst(leaf));
+ sb.append("\n");
+ }
+ sb.delete(sb.length() - "\n".length(), sb.length());
+ sb.delete(sb.length() - "\n".length(), sb.length());
+ return sb.toString();
}
-
- public void visit(LONotEqual ne) throws VisitorException {
- visitBinary(ne, "!=");
+
+ private String planString(LogicalPlan lp){
+ StringBuilder sb = new StringBuilder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ if(lp!=null)
+ lp.explain(baos, mStream);
+ else
+ return "";
+ sb.append(USep);
+ sb.append(shiftStringByTabs(baos.toString(), 2));
+ return sb.toString();
}
-
- public void visit(LOOr or) throws VisitorException {
- visitBinary(or, "OR");
+
+ private String planString(List<LogicalPlan> logicalPlanList){
+ StringBuilder sb = new StringBuilder();
+ if(logicalPlanList!=null)
+ for (LogicalPlan lp : logicalPlanList) {
+ sb.append(planString(lp));
+ }
+ return sb.toString();
}
- public void visit(LOProject p) throws VisitorException {
- print(p);
- if (p.isStar()) {
- mStream.print(" ALL ");
+ private String depthFirst(LogicalOperator node) throws VisitorException {
+ StringBuilder sb = new StringBuilder(node.name());
+ if(node instanceof ExpressionOperator) {
+ sb.append(" FieldSchema: ");
+ try {
+ sb.append(((ExpressionOperator)node).getFieldSchema());
+ } catch (Exception e) {
+ //sb.append("Caught Exception: " + e.getMessage());
+ }
} else {
- List<Integer> cols = p.getProjection();
- mStream.print(" COL");
- if (cols.size() > 1) mStream.print("S");
- mStream.print(" (");
- for (int i = 0; i < cols.size(); i++) {
- if (i > 0) mStream.print(", ");
- mStream.print(cols.get(i));
+ sb.append(" Schema: ");
+ try {
+ sb.append(node.getSchema());
+ } catch (Exception e) {
+ //sb.append("Caught exception: " + e.getMessage());
}
- mStream.print(")");
}
- mStream.print(" FROM ");
- if (p.getSentinel()) {
- // This project is connected to some other relation, don't follow
- // that path or we'll cycle in the graph.
- p.getExpression().name();
- } else {
- mIndent++;
- p.getExpression().visit(this);
- mIndent--;
+ sb.append(" Type: " + DataType.findTypeName(node.getType()));
+ sb.append("\n");
+ if(node instanceof LOFilter){
+ sb.append(planString(((LOFilter)node).getComparisonPlan()));
+ }
+ else if(node instanceof LOForEach){
+ sb.append(planString(((LOForEach)node).getForEachPlan()));
+ }
+ else if(node instanceof LOGenerate){
+ sb.append(planString(((LOGenerate)node).getGeneratePlans()));
+
+ }
+ else if(node instanceof LOCogroup){
+ MultiMap<LogicalOperator, LogicalPlan> plans =
((LOCogroup)node).getGroupByPlans();
+ for (LogicalOperator lo : plans.keySet()) {
+ // Visit the associated plans
+ for (LogicalPlan plan : plans.get(lo)) {
+ sb.append(planString(plan));
+ }
+ }
+ }
+ else if(node instanceof LOSort){
+ sb.append(planString(((LOSort)node).getSortColPlans()));
+ }
+ else if(node instanceof LOSplitOutput){
+ sb.append(planString(((LOSplitOutput)node).getConditionPlan()));
}
- }
-
- public void visit(LORegexp r) throws VisitorException {
- print(r);
- mStream.print(" REGEX (" + r.getRegexp() + ") LOOKING IN (");
- r.getOperand().visit(this);
- mStream.print(")");
- }
-
- /**
- * Only LOUnion.visit() should ever call this method.
- */
- /*
- @Override
- public void visitUnion(LOUnion u) {
- print(u, u.name());
- super.visitUnion(u);
- }
- /**
- * Only LOSort.visit() should ever call this method.
- */
- /*
- @Override
- public void visitSort(LOSort s) {
- List<EvalSpec> ls = new ArrayList<EvalSpec>();
- ls.add(s.getSpec());
- print(s, s.name());
- super.visitSort(s);
- }
+ List<LogicalOperator> predecessors = mPlan.getPredecessors(node);
+ if(node instanceof LOProject) {
+ System.err.println("LOProject " + node + " predecessors: " +
predecessors + " in plan " + mPlan);
+ System.err.println("mPlan size: " + mPlan.size());
+ } else if(node instanceof LOSort) {
+ System.err.println("LOSort : " + node + " predecessors: " +
predecessors + " in plan " + mPlan);
+ System.err.println("mPlan size: " + mPlan.size());
+ }
- /**
- * Only LOSplit.visit() should ever call this method.
- */
- /*
- @Override
- public void visitSplit(LOSplit s) {
- print(s, s.name());
- super.visitSplit(s);
- }
+ if (predecessors == null)
+ return sb.toString();
- /**
- * Only LOStore.visit() should ever call this method.
- */
- /*
- @Override
- public void visitStore(LOStore s) {
- print(s, s.name());
- super.visitStore(s);
- }
-
- private void print(LogicalOperator lo, String name) {
- List<EvalSpec> empty = new ArrayList<EvalSpec>();
- print(lo, name, empty);
- }
- */
-
- private void visitBinary(
- BinaryExpressionOperator b,
- String op) throws VisitorException {
- print(b);
- mStream.print(" (");
- b.getLhsOperand().visit(this);
- mStream.print(") " + op + " (");
- b.getRhsOperand().visit(this);
- mStream.print(") ");
- }
-
- private void visitUnary(
- UnaryExpressionOperator e,
- String op) throws VisitorException {
- print(e);
- mStream.print(op + " (");
- e.getOperand().visit(this);
- mStream.print(") ");
+ Collections.sort(predecessors);
+ int i = 0;
+ for (LogicalOperator pred : predecessors) {
+ i++;
+ String DFStr = depthFirst(pred);
+ if (DFStr != null) {
+ sb.append(LSep);
+ if (i < predecessors.size())
+ sb.append(shiftStringByTabs(DFStr, 2));
+ else
+ sb.append(shiftStringByTabs(DFStr, 1));
+ }
+ }
+ return sb.toString();
}
- private void print(LogicalOperator lo) {
- for (int i = 0; i < mIndent; i++) mStream.print(" ");
+ private String shiftStringByTabs(String DFStr, int TabType) {
+ StringBuilder sb = new StringBuilder();
+ String[] spl = DFStr.split("\n");
- printName(lo);
+ String tab = (TabType == 1) ? TAB1 : TABMore;
- if (!(lo instanceof ExpressionOperator)) {
- mStream.print("Inputs: ");
- for (LogicalOperator predecessor : mPlan.getPredecessors(lo)) {
- printName(predecessor);
- }
- mStream.print("Schema: ");
- try {
- printSchema(lo.getSchema());
- } catch (FrontendException fe) {
- // ignore it, nothing we can do
- mStream.print("()");
- }
+ sb.append(spl[0] + "\n");
+ for (int i = 1; i < spl.length; i++) {
+ sb.append(tab);
+ sb.append(spl[i]);
+ sb.append("\n");
}
- mStream.print(" : ");
- }
-
- private void printName(LogicalOperator lo) {
- mStream.println(lo.name() + " key(" + lo.getOperatorKey().scope +
- ", " + lo.getOperatorKey().id + ") ");
+ return sb.toString();
}
- private void printSchema(Schema schema) {
- mStream.print("(");
- for (Schema.FieldSchema fs : schema.getFields()) {
- if (fs.alias != null) mStream.print(fs.alias + ": ");
- mStream.print(DataType.findTypeName(fs.type));
- if (fs.schema != null) {
- if (fs.type == DataType.BAG) mStream.print("{");
- printSchema(fs.schema);
- if (fs.type == DataType.BAG) mStream.print("}");
- }
- }
- mStream.print(")");
+ private void dispTabs() {
+ for (int i = 0; i < levelCntr; i++)
+ System.out.print(TAB1);
}
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
Tue Jun 17 17:39:00 2008
@@ -148,7 +148,7 @@
@Override
public String name() {
- return "Project " + mKey.scope + "-" + mKey.id;
+ return "Project " + mKey.scope + "-" + mKey.id + " Projections: " +
(mIsStar? " [*] ": mProjection);
}
@Override
@@ -184,10 +184,10 @@
log.debug("expression operator schema: " +
expressionOperator.getSchema());
log.debug("expression operator type: " +
expressionOperator.getType());
//TODO
- //the type of the operator will be unkown. when
type checking is in place
+ //the type of the operator will be unknown. when
type checking is in place
//add the type of the operator as a parameter to
the fieldschema creation
- //mFieldSchema = new
Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema(), expressionOperator.getType());
- mFieldSchema = new
Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema());
+ mFieldSchema = new
Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema(), expressionOperator.getType());
+ //mFieldSchema = new
Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema());
}
} else {
log.warn("The input for a projection operator cannot
be null");
@@ -246,8 +246,13 @@
Schema.FieldSchema expOpFs =
((ExpressionOperator)expressionOperator).getFieldSchema();
if(null != expOpFs) {
Schema s = expOpFs.schema;
+ log.debug("Schema s: " + s);
if(null != s) {
- fss.add(s.getField(colNum));
+ if(colNum < s.size()) {
+ fss.add(s.getField(colNum));
+ } else {
+ fss.add(new
Schema.FieldSchema(null, DataType.BYTEARRAY));
+ }
} else {
fss.add(new Schema.FieldSchema(null,
DataType.BYTEARRAY));
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
Tue Jun 17 17:39:00 2008
@@ -100,12 +100,18 @@
if (!mIsSchemaComputed && (null == mSchema)) {
// get our parent's schema
Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+ ArrayList<Schema.FieldSchema> fss = new
ArrayList<Schema.FieldSchema>();
try {
LogicalOperator op = s.iterator().next();
if (null == op) {
throw new FrontendException("Could not find operator in
plan");
}
- mSchema = op.getSchema();
+ if(op instanceof ExpressionOperator) {
+ fss.add(((ExpressionOperator)op).getFieldSchema());
+ mSchema = new Schema(fss);
+ } else {
+ mSchema = op.getSchema();
+ }
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
mSchema = null;
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
Tue Jun 17 17:39:00 2008
@@ -19,6 +19,7 @@
import java.io.IOException;
import java.io.OutputStream;
+import java.io.PrintStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
@@ -27,6 +28,7 @@
import java.util.Map;
import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.VisitorException;
public class LogicalPlan extends OperatorPlan<LogicalOperator> {
private static final long serialVersionUID = 2L;
@@ -46,4 +48,17 @@
return getSingleLeafPlanOutputOp().getType() ;
}
+ public void explain(OutputStream out, PrintStream ps){
+ LOPrinter lpp = new LOPrinter(ps, this);
+
+ try {
+ lpp.print(out);
+ } catch (VisitorException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Tue Jun 17 17:39:00 2008
@@ -236,6 +236,10 @@
flattenedColumns.add(column);
(gis.get(i)).isInner = true;
projectPlan.add(column);
+ if(projectInput instanceof ExpressionOperator) {
+ projectPlan.add(projectInput);
+ projectPlan.connect(projectInput, column);
+ }
log.debug("parseCogroup: Added operator " +
column.getClass().getName() + " " + column + " to logical plan " + projectPlan);
generatePlans.add(projectPlan);
}
@@ -647,6 +651,7 @@
log.debug("Added operator: " + distinct.getClass().getName() +
" to the logical plan");
lp.connect(op, distinct);
log.debug("Connected alias: " + op.getAlias() + " operator " +
op.getClass().getName() + " to operator " + distinct.getClass().getName());
+ op = distinct;
})
| (<CROSS> op = CrossClause(lp))
| (<JOIN> op = JoinClause(lp))
@@ -1521,6 +1526,10 @@
project.setStar(true);
item = project;
lp.add(project);
+ if(input instanceof ExpressionOperator) {
+ lp.add(input);
+ lp.connect(input, project);
+ }
log.debug("FGItem: Added operator " +
project.getClass().getName() + " " + project + " to logical plan " + lp);
}
)
@@ -1728,7 +1737,6 @@
"." projection = BracketedSimpleProj(subSchema,lp,item)
{
assertAtomic(item,false);
- lp.remove(item);
item = projection;
}
)
@@ -1892,6 +1900,10 @@
project.setStar(true);
item = project;
lp.add(project);
+ if(input instanceof ExpressionOperator) {
+ lp.add(input);
+ lp.connect(input, project);
+ }
log.debug("EvalArgsItem: Added operator " +
project.getClass().getName() + " " + project + " to logical plan " + lp);
}
)
@@ -2215,6 +2227,10 @@
ExpressionOperator project = new LOProject(lp, new
OperatorKey(scope, getNextId()), eOp, colList);
lp.add(project);
log.debug("SimpleProj: Added operator " +
project.getClass().getName() + " " + project + " to logical plan " + lp);
+ if(eOp instanceof ExpressionOperator) {
+ lp.add(eOp);
+ lp.connect(eOp, project);
+ }
log.trace("Exiting SimpleProj");
return project;
}
@@ -2389,6 +2405,10 @@
log.debug("eOp: " + eOp.getClass().getName() + " " +
eOp);
lp.add(project);
log.debug("DollarVar: Added operator " +
project.getClass().getName() + " " + project + " to logical plan " + lp);
+ if(eOp instanceof ExpressionOperator) {
+ lp.add(eOp);
+ lp.connect(eOp, project);
+ }
} catch (Exception planException) {
ParseException parseException = new
ParseException(planException.getMessage());
throw parseException;
@@ -2429,6 +2449,10 @@
try {
lp.add(item);
log.debug("AliasFieldOrSpec: Added
operator " + item.getClass().getName() + " " + item + " to logical plan " + lp);
+ if(op instanceof ExpressionOperator) {
+ lp.add(op);
+ lp.connect(op, item);
+ }
} catch (Exception planException) {
ParseException parseException = new
ParseException(planException.getMessage());
throw parseException;
@@ -2457,6 +2481,10 @@
try {
lp.add(item);
log.debug("AliasFieldOrSpec: Added operator " +
item.getClass().getName() + " " + item + " to logical plan " + lp);
+ if(eOp instanceof ExpressionOperator) {
+ lp.add(eOp);
+ lp.connect(eOp, item);
+ }
} catch (Exception planException) {
ParseException parseException = new
ParseException(planException.getMessage());
throw parseException;
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
Tue Jun 17 17:39:00 2008
@@ -93,6 +93,9 @@
else if (eOp instanceof LONot) {
visit((LONot) eOp) ;
}
+ else if (eOp instanceof LOMapLookup) {
+ visit((LOMapLookup) eOp) ;
+ }
// TODO: Check that all operators are included here
}
@@ -275,6 +278,19 @@
}
+ @Override
+ public void visit(LOMapLookup map)
+ throws VisitorException {
+ if(!DataType.isAtomic(DataType.findType(map.getLookUpKey()))) {
+ String msg = "Map key should be a basic type" ;
+ msgCollector.collect(msg, MessageType.Error);
+ throw new VisitorException(msg) ;
+ }
+
+ map.setType(map.getValueType());
+
+ }
+
/**
* LORegexp expects CharArray as input
* Itself always returns Boolean
@@ -957,6 +973,11 @@
}
}
+ @Override
+ public void visit(LOIsNull uniOp) throws VisitorException {
+ uniOp.setType(DataType.BOOLEAN) ;
+ }
+
private void insertLeftCastForBinaryOp(BinaryExpressionOperator binOp,
byte toType ) {
LogicalPlan currentPlan = (LogicalPlan) mCurrentWalker.getPlan() ;
@@ -1983,8 +2004,8 @@
outputSchemaIdx++ ;
}
} else {
- schema.getField(outputSchemaIdx).type =
leaf.getType() ;
- schema.getField(outputSchemaIdx).schema =
leaf.getSchema() ;
+ schema.getField(outputSchemaIdx).type =
DataType.BYTEARRAY;
+ schema.getField(outputSchemaIdx).schema = null;
outputSchemaIdx++ ;
}
}
@@ -1996,8 +2017,8 @@
}
}
else {
- throw new AssertionError("Unsupported inner plan type"
- + " in LOGenerate" ) ;
+ throw new AssertionError("Unsupported inner plan type:
"
+ +
DataType.findTypeName(leaf.getType()) + " in LOGenerate" ) ;
}
}
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
---
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
(original)
+++
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Tue Jun 17 17:39:00 2008
@@ -53,6 +53,7 @@
import org.apache.pig.impl.logicalLayer.ExpressionOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
+import org.apache.pig.impl.logicalLayer.LOPrinter;
import org.apache.pig.impl.logicalLayer.schema.Schema;
@@ -881,6 +882,16 @@
buildPlan("e = foreach c generate $0, ($1 IS NOT NULL ? $1 - 5 : 0);");
}
+ private void printPlan(LogicalPlan lp) {
+ LOPrinter graphPrinter = new LOPrinter(System.err, lp);
+ System.err.println("Printing the logical plan");
+ try {
+ graphPrinter.visit();
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ }
+ }
+
// Helper Functions
// Helper Functions
