Author: gates Date: Mon May 19 12:45:33 2008 New Revision: 657939 URL: http://svn.apache.org/viewvc?rev=657939&view=rev Log: Make it all compile. With this patch we can now return the build.xml compile-sources rule to its previous state of building all java files below src. There are a number of places where code is commented out (all marked TODO FIX). These will be addressed over the next few days as we tie together loose ends.
All non-map reduce unit tests also now run, and pass. Removed: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/POMapreduce.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/SplitSpec.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapreduceExec/MapReduceLauncher.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapreduceExec/PigCombine.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapreduceExec/PigInputFormat.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapreduceExec/PigMapReduce.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapreduceExec/PigOutputFormat.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapreduceExec/PigSplit.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POCogroup.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POEval.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POLoad.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POSort.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POSplit.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POStore.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/POUnion.java incubator/pig/branches/types/src/org/apache/pig/impl/builtin/ADD.java incubator/pig/branches/types/src/org/apache/pig/impl/builtin/DIVIDE.java incubator/pig/branches/types/src/org/apache/pig/impl/builtin/MULTIPLY.java incubator/pig/branches/types/src/org/apache/pig/impl/builtin/SUBTRACT.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/AndCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/CompCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/Cond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/FalseCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/FuncCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/NotCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/OrCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/RegexpCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/cond/TrueCond.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/window/TimeWindowSpec.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/window/TupleWindowSpec.java incubator/pig/branches/types/src/org/apache/pig/impl/eval/window/WindowSpec.java Modified: incubator/pig/branches/types/build.xml incubator/pig/branches/types/src/org/apache/pig/Main.java incubator/pig/branches/types/src/org/apache/pig/PigServer.java incubator/pig/branches/types/src/org/apache/pig/StandAloneParser.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java incubator/pig/branches/types/src/org/apache/pig/builtin/TOKENIZE.java incubator/pig/branches/types/src/org/apache/pig/impl/builtin/ShellBagEvalFunc.java incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POPrinter.java incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POVisitor.java incubator/pig/branches/types/test/org/apache/pig/test/NonDefaultBagFactory.java incubator/pig/branches/types/test/org/apache/pig/test/TestDataBag.java incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java Modified: incubator/pig/branches/types/build.xml URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/build.xml (original) +++ incubator/pig/branches/types/build.xml Mon May 19 12:45:33 2008 @@ -133,32 +133,9 @@ <target name="compile-sources"> <javac encoding="${build.encoding}" srcdir="${sources}" - includes="**/plan/*.java, **/plan/optimizer/*.java, **/data/*.java, **/pig/builtin/*.java, - **/impl/io/*.java, **/impl/mapReduceLayer/*.java, - **/test/utils/*.java, **/test/TestOperatorPlan.java, **/test/TestBuiltin.java, - **/test/TestConstExpr.java, **/test/TestFilter.java, **/test/TestPhyOp.java, - **/test/TestAdd.java, **/test/TestSubtract.java, **/test/TestMultiply.java, - **/test/TestDivide.java, **/test/TestMod.java, **/test/TestGreaterThan.java, - **/test/TestGTOrEqual.java,**/test/TestLessThan.java,**/test/TestLTOrEqual.java, - **/test/TestEqualTo.java,**/test/TestNotEqualTo.java, **/test/TestPOGenerate.java, - **/test/TestProject.java, **/test/TestLoad.java, **/test/TestStore.java, - **/test/FakeFSOutputStream.java, **/test/TestPackage.java, **/test/TestForEach.java, - **/test/TestLocalRearrange.java, **/test/TestPOUserFunc.java, - **/test/TestPODistinct.java, **/test/TestPOSort.java, **/test/TestPOCast.java, - **/test/TestSchema.java, **/test/TestLogicalPlanBuilder.java,**/test/TestUnion.java, **/test/TestMRCompiler.java, - **/test/FakeFSInputStream.java, **/test/Util.java, **/test/TestJobSubmission.java, - **/test/TestLocalJobSubmission.java, **/test/TestPOMapLookUp.java, - **/test/TestPOBinCond.java, **/test/TestPONegative.java, **/pig/impl/builtin/GFCross.java, - **/logicalLayer/*.java, **/logicalLayer/parser/NodeIdGenerator.java, - **/logicalLayer/schema/*.java, **/physicalLayer/topLevelOperators/*.java, - **/physicalLayer/topLevelOperators/**/*.java, **/physicalLayer/plans/*.java, - **/physicalLayer/Result.java, **/mapReduceLayer/**/*.java, - **/physicalLayer/POStatus.java, **/bzip2r/*.java, - **/validators/*.java , **/test/TestInputOutputFileValidator.java , - **/test/TestTypeCheckingValidator.java, - **/backend/executionengine/ExecException.java, - **/tools/grunt/*.java, **/test/TestGrunt.java" - destdir="${dist}" debug="${javac.debug}" optimize="${javac.optimize}" target="${javac.version}" source="${javac.version}" deprecation="${javac.deprecation}"> + includes="**/*.java" destdir="${dist}" debug="${javac.debug}" + optimize="${javac.optimize}" target="${javac.version}" + source="${javac.version}" deprecation="${javac.deprecation}"> <compilerarg line="${javac.args} ${javac.args.warnings}" /> <classpath refid="${cp}" /> </javac> @@ -245,6 +222,7 @@ <batchtest fork="yes" todir="${test.log.dir}" unless="testcase"> <fileset dir="test"> + <!-- <include name="**/TestBuiltin.java" /> <include name="**/TestOperatorPlan.java" /> <include name="**/TestPhyOp.java" /> @@ -284,13 +262,26 @@ <include name="**/TestPONegative.java" /> <include name="**/TestGrunt.java" /> <include name="**/TestPOCast.java" /> - <!-- + --> <include name="**/*Test*.java" /> + <!-- Excluced because they are end-to-end, don't work yet. --> + <exclude name="**/TestAlgebraicEval.java" /> + <exclude name="**/TestCombiner.java" /> + <exclude name="**/TestCompressedFiles.java" /> + <exclude name="**/TestEvalPipeline.java" /> + <exclude name="**/TestFilterOpNumeric.java" /> + <exclude name="**/TestFilterOpString.java" /> + <exclude name="**/TestInfixArithmetic.java" /> + <exclude name="**/TestMapReduce.java" /> + <exclude name="**/TestPigFile.java" /> + <exclude name="**/TestPigSplit.java" /> + <exclude name="**/TestStoreOld.java" /> + <!-- Excluced because we don't want to run them --> + <exclude name="**/TestHelper.java" /> <exclude name="**/TestLargeFile.java" /> <exclude name="**/TestOrderBy.java" /> <exclude name="**/TestPi.java" /> <exclude name="**/nightly/**" /> - --> </fileset> </batchtest> </junit> Modified: incubator/pig/branches/types/src/org/apache/pig/Main.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/Main.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/Main.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/Main.java Mon May 19 12:45:33 2008 @@ -32,7 +32,7 @@ import org.apache.log4j.ConsoleAppender; import org.apache.log4j.PatternLayout; import org.apache.log4j.PropertyConfigurator; -import org.apache.pig.PigServer.ExecType; +import org.apache.pig.ExecType; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder; import org.apache.pig.tools.cmdline.CmdLineParser; Modified: incubator/pig/branches/types/src/org/apache/pig/PigServer.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/PigServer.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/PigServer.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/PigServer.java Mon May 19 12:45:33 2008 @@ -45,9 +45,14 @@ import org.apache.pig.data.Tuple; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.io.FileLocalizer; +import org.apache.pig.impl.logicalLayer.ExpressionOperator; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.LogicalOperator; import org.apache.pig.impl.logicalLayer.LogicalPlan; +import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder; import org.apache.pig.impl.logicalLayer.OperatorKey; +import org.apache.pig.impl.logicalLayer.parser.ParseException; +import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.WrappedIOException; @@ -209,34 +214,30 @@ return; } - // TODO FIX Need to change so that only syntax parsing is done here, and so that logical plan is additive. - // parse the query into a logical plan LogicalPlan lp = null; -// TODO FIX Need to uncomment this with the right logic - /*try { - lp = (new LogicalPlanBuilder(pigContext).parse(scope, query, aliases, opTable)); + Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>(); + Map<String, ExpressionOperator> defineAliases = new HashMap<String, ExpressionOperator>(); + try { + lp = (new LogicalPlanBuilder(pigContext).parse(scope, query, + aliases, opTable, aliasOp, defineAliases)); } catch (ParseException e) { throw (IOException) new IOException(e.getMessage()).initCause(e); - }*/ - - /* - if (lp.getAlias() != null) { - aliases.put(lp.getAlias(), lp); } - */ } public void dumpSchema(String alias) throws IOException{ - // TODO FIX Need to rework so we can get an appropriate output schema - /* LogicalPlan lp = aliases.get(alias); if (lp == null) throw new IOException("Invalid alias - " + alias); - TupleSchema schema = lp.getOpTable().get(lp.getRoot()).outputSchema(); - - System.out.println(schema.toString()); - */ + try { + Schema schema = lp.getLeaves().get(0).getSchema(); + System.out.println(schema.toString()); + } catch (FrontendException fe) { + IOException ioe = new IOException(fe.getMessage()); + ioe.initCause(fe); + throw ioe; + } } public void setJobName(String name){ @@ -303,7 +304,6 @@ } public void store(LogicalPlan readFrom, String filename, String func) throws IOException { - // TODO FIX /* LogicalPlan storePlan = QueryParser.generateStorePlan(readFrom.getOpTable(), scope, Modified: incubator/pig/branches/types/src/org/apache/pig/StandAloneParser.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/StandAloneParser.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/StandAloneParser.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/StandAloneParser.java Mon May 19 12:45:33 2008 @@ -24,8 +24,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.pig.PigServer.ExecType; +import org.apache.pig.ExecType; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.LogicalPlan; public class StandAloneParser { @@ -68,12 +69,14 @@ for (Iterator<String> it = pig.getAliases().keySet().iterator(); it.hasNext(); ) { String alias = it.next(); LogicalPlan lp = pig.getAliases().get(alias); - System.out.print(alias + "->" + lp.getOpTable().get(lp.getRoot()).outputSchema()); + System.out.print(alias + "->" + lp.getLeaves().get(0).getSchema()); if (it.hasNext()) System.out.print(", \n"); else System.out.print("\n"); } } catch (IOException e) { log.error(e); + } catch (FrontendException fe) { + log.error(fe); } } } Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java Mon May 19 12:45:33 2008 @@ -54,9 +54,12 @@ } public void explain(OutputStream out) { + // TODO FIX + /* POVisitor lprinter = new POPrinter(opTable, new PrintStream(out)); ((PhysicalOperator)opTable.get(root)).visit(lprinter); + */ } public Map<OperatorKey, ExecPhysicalOperator> getOpTable() { Modified: incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java Mon May 19 12:45:33 2008 @@ -42,7 +42,6 @@ import org.apache.pig.impl.physicalLayer.PhysicalOperator; import org.apache.pig.impl.logicalLayer.LogicalPlan; import org.apache.pig.impl.logicalLayer.parser.NodeIdGenerator; -import org.apache.pig.impl.eval.cond.Cond; import org.apache.pig.impl.io.FileSpec; import java.util.Iterator; @@ -104,11 +103,15 @@ throw new ExecException("No Plan to compile"); } - return compile(new ExecLogicalPlan[]{ plan } , properties); + // TODO FIX + // return compile(new ExecLogicalPlan[]{ plan } , properties); + return null; } public LocalPhysicalPlan compile(LogicalPlan[] plans, Properties properties) throws ExecException { + // TODO FIX + /* if (plans == null) { throw new ExecException("No Plans to compile"); } @@ -133,6 +136,8 @@ } return new LocalPhysicalPlan(physicalKey, physicalOpTable); + */ + return null; } public LocalJob execute(ExecPhysicalPlan plan) throws ExecException { @@ -177,6 +182,8 @@ Properties properties) throws ExecException { + // TODO FIX + /* LocalResult materializedResult = materializedResults.get(logicalKey); if (materializedResult != null) { @@ -202,6 +209,8 @@ } return physicalKey; + */ + return null; } private boolean compileOperator(OperatorKey logicalKey, @@ -209,9 +218,11 @@ Properties properties, OperatorKey physicalKey) throws ExecException { + // TODO FIX + /* ExecPhysicalOperator pp; LogicalOperator lo = logicalOpTable.get(logicalKey); - String scope = lo.getScope(); + String scope = lo.getOperatorKey().getScope(); boolean compileInputs = true; if (lo instanceof LOEval) { @@ -287,6 +298,8 @@ logicalToPhysicalKeys.put(logicalKey, physicalKey); return compileInputs; + */ + return false; } } Modified: incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java Mon May 19 12:45:33 2008 @@ -53,9 +53,12 @@ } public void explain(OutputStream out) { + // TODO FIX + /* POVisitor lprinter = new POPrinter(opTable, new PrintStream(out)); ((PhysicalOperator)opTable.get(root)).visit(lprinter); + */ } public Map<OperatorKey, ExecPhysicalOperator> getOpTable() { Modified: incubator/pig/branches/types/src/org/apache/pig/builtin/TOKENIZE.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/builtin/TOKENIZE.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/builtin/TOKENIZE.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/builtin/TOKENIZE.java Mon May 19 12:45:33 2008 @@ -24,6 +24,7 @@ import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.data.BagFactory; import org.apache.pig.data.DataBag; +import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.impl.logicalLayer.schema.Schema; @@ -52,12 +53,8 @@ @Override public Schema outputSchema(Schema input) { - // TODO FIX - /* - TupleSchema schema = new TupleSchema(); - schema.add(new AtomSchema("token")); + Schema schema = new Schema(new Schema.FieldSchema("token", + DataType.CHARARRAY)); return schema; - */ - return null; } } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/builtin/ShellBagEvalFunc.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/builtin/ShellBagEvalFunc.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/builtin/ShellBagEvalFunc.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/builtin/ShellBagEvalFunc.java Mon May 19 12:45:33 2008 @@ -34,6 +34,7 @@ import org.apache.pig.data.DefaultAbstractBag; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; +import org.apache.pig.backend.executionengine.ExecException; public class ShellBagEvalFunc extends EvalFunc<DataBag> { @@ -123,10 +124,17 @@ if (os == null) { startProcess(); } - os.write(input.toDelimitedString(fieldDelimString).getBytes()); - os.write(recordDelim); - os.write(groupDelim); - os.flush(); + try { + os.write(input.toDelimitedString(fieldDelimString).getBytes()); + os.write(recordDelim); + os.write(groupDelim); + os.flush(); + } catch (ExecException ee) { + IOException ioe = new IOException(ee.getMessage()); + ioe.initCause(ee); + throw ioe; + } + try{ bags.put(output); }catch(InterruptedException e){} Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Mon May 19 12:45:33 2008 @@ -42,7 +42,7 @@ */ public void explain(OutputStream out) { - // TODO + // TODO FIX /* LOVisitor lprinter = new LOPrinter(new PrintStream(out)); Modified: incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POPrinter.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POPrinter.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POPrinter.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POPrinter.java Mon May 19 12:45:33 2008 @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Iterator; +/* import org.apache.pig.backend.hadoop.executionengine.POMapreduce; import org.apache.pig.backend.local.executionengine.POCogroup; import org.apache.pig.backend.local.executionengine.POEval; @@ -36,10 +37,13 @@ import org.apache.pig.impl.eval.EvalSpecPrinter; import org.apache.pig.impl.io.FileSpec; import org.apache.pig.impl.logicalLayer.OperatorKey; +*/ public class POPrinter extends POVisitor { + // TODO FIX + /* private PrintStream mStream = null; public POPrinter(Map<OperatorKey, ExecPhysicalOperator> opTable, @@ -48,10 +52,6 @@ mStream = ps; } - /** - * Only POMapreduce.visit() and subclass implementations of this function - * should ever call this method. - */ public void visitMapreduce(POMapreduce mr) { mStream.println("MAPREDUCE"); printHeader(mr); @@ -95,30 +95,18 @@ super.visitMapreduce(mr); } - /** - * Only POLoad.visit() and subclass implementations of this function - * should ever call this method. - */ public void visitLoad(POLoad load) { mStream.println("LOAD"); printHeader(load); super.visitLoad(load); } - /** - * Only POSort.visit() and subclass implementations of this function - * should ever call this method. - */ public void visitSort(POSort s) { mStream.println("SORT"); printHeader(s); super.visitSort(s); } - /** - * Only POStore.visit() and subclass implementations of this function - * should ever call this method. - */ public void visitStore(POStore s) { mStream.println("STORE"); printHeader(s); @@ -141,6 +129,7 @@ } mStream.println(); } + */ } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POVisitor.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POVisitor.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POVisitor.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/physicalLayer/POVisitor.java Mon May 19 12:45:33 2008 @@ -19,6 +19,7 @@ import java.util.Map; +/* import org.apache.pig.backend.hadoop.executionengine.POMapreduce; import org.apache.pig.backend.local.executionengine.POCogroup; import org.apache.pig.backend.local.executionengine.POEval; @@ -29,6 +30,7 @@ import org.apache.pig.backend.local.executionengine.POUnion; import org.apache.pig.backend.executionengine.ExecPhysicalOperator; import org.apache.pig.impl.logicalLayer.OperatorKey; +*/ /** * A visitor mechanism for navigating and operating on a tree of Physical @@ -52,12 +54,17 @@ * These methods are only public to make them accessible to the PO* objects. */ abstract public class POVisitor { + // TODO FIX This whole class needs to be removed (it's been superceded + // by PhysOpVisitor) but not yet as there are other classes + // extending it. + /* protected Map<OperatorKey, ExecPhysicalOperator> mOpTable; /** * @param opTable Operator table for the physical plan. */ + /* protected POVisitor(Map<OperatorKey, ExecPhysicalOperator> opTable) { mOpTable = opTable; } @@ -66,6 +73,7 @@ * Only POMapreduce.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitMapreduce(POMapreduce mr) { basicVisit(mr); } @@ -74,6 +82,7 @@ * Only POLoad.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitLoad(POLoad load) { basicVisit(load); } @@ -82,6 +91,7 @@ * Only POSort.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitSort(POSort s) { basicVisit(s); } @@ -90,6 +100,7 @@ * Only POStore.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitStore(POStore s) { basicVisit(s); } @@ -98,6 +109,7 @@ * Only POCogroup.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitCogroup(POCogroup g) { basicVisit(g); } @@ -106,6 +118,7 @@ * Only POEval.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitEval(POEval e) { basicVisit(e); } @@ -114,6 +127,7 @@ * Only POSplit.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitSplit(POSplit s) { basicVisit(s); } @@ -122,6 +136,7 @@ * Only POUnion.visit() and subclass implementations of this function * should ever call this method. */ + /* public void visitUnion(POUnion u) { basicVisit(u); } @@ -131,6 +146,7 @@ ((PhysicalOperator)mOpTable.get(po.inputs[i])).visit(this); } } + */ } Modified: incubator/pig/branches/types/test/org/apache/pig/test/NonDefaultBagFactory.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/NonDefaultBagFactory.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/NonDefaultBagFactory.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/NonDefaultBagFactory.java Mon May 19 12:45:33 2008 @@ -17,14 +17,15 @@ */ package org.apache.pig.test; +import java.util.Comparator; + import org.apache.pig.data.*; -import org.apache.pig.impl.eval.EvalSpec; // Test data bag factory, for testing that we can propery provide a non // default bag factory. public class NonDefaultBagFactory extends BagFactory { public DataBag newDefaultBag() { return null; } - public DataBag newSortedBag(EvalSpec sortSpec) { return null; } + public DataBag newSortedBag(Comparator<Tuple> comp) { return null; } public DataBag newDistinctBag() { return null; } public NonDefaultBagFactory() { super(); } Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestDataBag.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestDataBag.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestDataBag.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestDataBag.java Mon May 19 12:45:33 2008 @@ -17,23 +17,12 @@ */ package org.apache.pig.test; -/* -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.util.Iterator; -import java.util.Random; -*/ - import java.util.*; import java.io.IOException; import org.junit.Test; import org.apache.pig.data.*; -import org.apache.pig.impl.eval.*; import org.apache.pig.impl.util.Spillable; /** @@ -74,8 +63,8 @@ return bag; } - public DataBag newSortedBag(EvalSpec sortSpec) { - DataBag bag = new SortedDataBag(sortSpec); + public DataBag newSortedBag(Comparator<Tuple> comp) { + DataBag bag = new SortedDataBag(comp); mMemMgr.register(bag); return bag; } Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java Mon May 19 12:45:33 2008 @@ -34,6 +34,7 @@ import org.apache.pig.EvalFunc; import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.builtin.BinStorage; import org.apache.pig.builtin.PigStorage; import org.apache.pig.builtin.TextLoader; @@ -172,22 +173,28 @@ @Override public DataBag exec(Tuple input) throws IOException { - DataBag output = BagFactory.getInstance().newDefaultBag(); - String str = input.get(0).toString(); + try { + DataBag output = BagFactory.getInstance().newDefaultBag(); + String str = input.get(0).toString(); - String title = str; + String title = str; - if (title != null) { - List<String> nGrams = makeNGrams(title); - - for (Iterator<String> it = nGrams.iterator(); it.hasNext(); ) { - Tuple t = TupleFactory.getInstance().newTuple(1); - t.set(0, it.next()); - output.add(t); + if (title != null) { + List<String> nGrams = makeNGrams(title); + + for (Iterator<String> it = nGrams.iterator(); it.hasNext(); ) { + Tuple t = TupleFactory.getInstance().newTuple(1); + t.set(0, it.next()); + output.add(t); + } } + + return output; + } catch (ExecException ee) { + IOException ioe = new IOException(ee.getMessage()); + ioe.initCause(ee); + throw ioe; } - - return output; } Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java Mon May 19 12:45:33 2008 @@ -25,7 +25,10 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintStream; +import java.net.URL; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import junit.framework.TestCase; @@ -35,16 +38,18 @@ import org.apache.pig.LoadFunc; import org.apache.pig.PigServer; import org.apache.pig.StoreFunc; -import org.apache.pig.PigServer.ExecType; +import org.apache.pig.ExecType; +import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.builtin.COUNT; import org.apache.pig.data.BagFactory; import org.apache.pig.data.DataBag; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; +import org.apache.pig.impl.PigContext; import org.apache.pig.impl.io.FileLocalizer; import org.apache.pig.impl.io.BufferedPositionedInputStream; -import org.apache.pig.impl.PigContext; +import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.backend.datastorage.ElementDescriptor; public class TestMapReduce extends TestCase { @@ -80,25 +85,37 @@ } @Override public DataBag exec(Tuple input) throws IOException { - DataBag output = BagFactory.getInstance().newDefaultBag(); - Iterator<Tuple> it = (DataType.toBag(input.get(0))).iterator(); - while(it.hasNext()) { - Tuple t = it.next(); - Tuple newT = TupleFactory.getInstance().newTuple(2); - newT.set(0, field0); - newT.set(1, t.get(0).toString()); - output.add(newT); - } + try { + DataBag output = BagFactory.getInstance().newDefaultBag(); + Iterator<Tuple> it = (DataType.toBag(input.get(0))).iterator(); + while(it.hasNext()) { + Tuple t = it.next(); + Tuple newT = TupleFactory.getInstance().newTuple(2); + newT.set(0, field0); + newT.set(1, t.get(0).toString()); + output.add(newT); + } - return output; + return output; + } catch (ExecException ee) { + IOException ioe = new IOException(ee.getMessage()); + ioe.initCause(ee); + throw ioe; + } } } static public class MyGroup extends EvalFunc<Tuple> { @Override public Tuple exec(Tuple input) throws IOException{ - Tuple output = TupleFactory.getInstance().newTuple(1); - output.set(0, new String("g")); - return output; + try { + Tuple output = TupleFactory.getInstance().newTuple(1); + output.set(0, new String("g")); + return output; + } catch (ExecException ee) { + IOException ioe = new IOException(ee.getMessage()); + ioe.initCause(ee); + throw ioe; + } } } static public class MyStorage implements LoadFunc, StoreFunc { @@ -121,8 +138,57 @@ } public void putNext(Tuple f) throws IOException { - os.write((f.toDelimitedString("-")+"\n").getBytes()); + try { + os.write((f.toDelimitedString("-")+"\n").getBytes()); + } catch (ExecException ee) { + IOException ioe = new IOException(ee.getMessage()); + ioe.initCause(ee); + throw ioe; + } + } + + public Boolean bytesToBoolean(byte[] b) throws IOException { + return false; + } + + public Integer bytesToInteger(byte[] b) throws IOException { + return 0; + } + + public Long bytesToLong(byte[] b) throws IOException { + return 0L; + } + + public Float bytesToFloat(byte[] b) throws IOException { + return 0.0f; + } + + public Double bytesToDouble(byte[] b) throws IOException { + return 0.0; } + + public String bytesToCharArray(byte[] b) throws IOException { + return ""; + } + + public Map<Object, Object> bytesToMap(byte[] b) throws IOException { + return new HashMap<Object, Object>(); + } + + public Tuple bytesToTuple(byte[] b) throws IOException { + return null; + } + + public DataBag bytesToBag(byte[] b) throws IOException { + return null; + } + + public void fieldsToRead(Schema schema) {} + + public Schema determineSchema(URL fileName) throws IOException { + return null; + } + } @Test public void testStoreFunction() throws Throwable { Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java?rev=657939&r1=657938&r2=657939&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java Mon May 19 12:45:33 2008 @@ -35,10 +35,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.builtin.BinStorage; import org.apache.pig.builtin.PigStorage; import org.apache.pig.data.*; -import org.apache.pig.PigServer.ExecType; +import org.apache.pig.ExecType; import org.apache.pig.impl.io.PigFile; import org.apache.pig.impl.PigContext; @@ -140,11 +141,17 @@ private Tuple getRandomTuple(int nestingLevel) throws IOException{ - int cardinality = rand.nextInt(2)+1; - Tuple t = TupleFactory.getInstance().newTuple(cardinality); - for (int i=0; i<cardinality; i++) - t.set(i, getRandomDatum(nestingLevel+1)); - return t; + try { + int cardinality = rand.nextInt(2)+1; + Tuple t = TupleFactory.getInstance().newTuple(cardinality); + for (int i=0; i<cardinality; i++) + t.set(i, getRandomDatum(nestingLevel+1)); + return t; + } catch (ExecException ee) { + IOException ioe = new IOException(ee.getMessage()); + ioe.initCause(ee); + throw ioe; + } } private DataBag getRandomBag(int maxCardinality, int nestingLevel) throws IOException{