Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMapReduce.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMapReduce.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMapReduce.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMapReduce.java Wed Nov 18 01:03:14 2009 @@ -46,6 +46,7 @@ import org.apache.pig.ExecType; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.builtin.COUNT; +import org.apache.pig.builtin.PigStorage; import org.apache.pig.data.BagFactory; import org.apache.pig.data.DataBag; import org.apache.pig.data.DataType; @@ -193,7 +194,7 @@ } } - static public class MyStorage implements LoadFunc, StoreFunc { + static public class MyStorage extends PigStorage { final static int COUNT = 10; @@ -202,9 +203,16 @@ public void setNulls(boolean hasNulls ) { this.hasNulls=hasNulls; } - public void bindTo(String fileName, BufferedPositionedInputStream is, long offset, long end) throws IOException { + /** + * + */ + public MyStorage() { + // initialize delimiter to be "-" for output + // since that is the delimiter in the tests below + super("-"); } - + + @Override public Tuple getNext() throws IOException { if (count < COUNT) { @@ -216,114 +224,6 @@ return null; } - - OutputStream os; - public void bindTo(OutputStream os) throws IOException { - this.os = os; - } - - public void finish() throws IOException { - - } - - public void putNext(Tuple f) throws IOException { - try { - os.write((f.toDelimitedString("-")+"\n").getBytes()); - } catch (ExecException ee) { - IOException ioe = new IOException(ee.getMessage()); - ioe.initCause(ee); - throw ioe; - } - } - - public Boolean bytesToBoolean(byte[] b) throws IOException { - return false; - } - - public Integer bytesToInteger(byte[] b) throws IOException { - return 0; - } - - public Long bytesToLong(byte[] b) throws IOException { - return 0L; - } - - public Float bytesToFloat(byte[] b) throws IOException { - return 0.0f; - } - - public Double bytesToDouble(byte[] b) throws IOException { - return 0.0; - } - - public String bytesToCharArray(byte[] b) throws IOException { - return ""; - } - - public Map<String, Object> bytesToMap(byte[] b) throws IOException { - return new HashMap<String, Object>(); - } - - public Tuple bytesToTuple(byte[] b) throws IOException { - return null; - } - - public DataBag bytesToBag(byte[] b) throws IOException { - return null; - } - - public void fieldsToRead(Schema schema) {} - - public byte[] toBytes(DataBag bag) throws IOException { - return null; - } - - public byte[] toBytes(String s) throws IOException { - return s.getBytes(); - } - - public byte[] toBytes(Double d) throws IOException { - return d.toString().getBytes(); - } - - public byte[] toBytes(Float f) throws IOException { - return f.toString().getBytes(); - } - - public byte[] toBytes(Integer i) throws IOException { - return i.toString().getBytes(); - } - - public byte[] toBytes(Long l) throws IOException { - return l.toString().getBytes(); - } - - public byte[] toBytes(Map<String, Object> m) throws IOException { - return m.toString().getBytes(); - } - - public byte[] toBytes(Tuple t) throws IOException { - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#determineSchema(java.lang.String, org.apache.pig.ExecType, org.apache.pig.backend.datastorage.DataStorage) - */ - public Schema determineSchema(String fileName, ExecType execType, - DataStorage storage) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.StoreFunc#getStorePreparationClass() - */ - @Override - public Class getStorePreparationClass() throws IOException { - // TODO Auto-generated method stub - return null; - } - }
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java Wed Nov 18 01:03:14 2009 @@ -24,12 +24,18 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordReader; import org.apache.pig.ExecType; import org.apache.pig.IndexableLoadFunc; +import org.apache.pig.LoadCaster; import org.apache.pig.PigException; import org.apache.pig.PigServer; import org.apache.pig.backend.datastorage.DataStorage; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan; import org.apache.pig.data.BagFactory; import org.apache.pig.data.DataBag; @@ -571,120 +577,66 @@ } /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bindTo(java.lang.String, org.apache.pig.impl.io.BufferedPositionedInputStream, long, long) - */ - @Override - public void bindTo(String fileName, BufferedPositionedInputStream is, - long offset, long end) throws IOException { - // TODO Auto-generated method stub - - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToBag(byte[]) - */ - @Override - public DataBag bytesToBag(byte[] b) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToCharArray(byte[]) - */ - @Override - public String bytesToCharArray(byte[] b) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToDouble(byte[]) - */ - @Override - public Double bytesToDouble(byte[] b) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToFloat(byte[]) - */ - @Override - public Float bytesToFloat(byte[] b) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToInteger(byte[]) - */ - @Override - public Integer bytesToInteger(byte[] b) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToLong(byte[]) + * @see org.apache.pig.LoadFunc#getNext() */ @Override - public Long bytesToLong(byte[] b) throws IOException { + public Tuple getNext() throws IOException { // TODO Auto-generated method stub return null; } /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToMap(byte[]) + * @see org.apache.pig.IndexableLoadFunc#initialize(org.apache.hadoop.conf.Configuration) */ @Override - public Map<String, Object> bytesToMap(byte[] b) throws IOException { + public void initialize(Configuration conf) throws IOException { // TODO Auto-generated method stub - return null; + } /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#bytesToTuple(byte[]) + * @see org.apache.pig.LoadFunc#getInputFormat() */ @Override - public Tuple bytesToTuple(byte[] b) throws IOException { + public InputFormat getInputFormat() throws IOException { // TODO Auto-generated method stub return null; } /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#determineSchema(java.lang.String, org.apache.pig.ExecType, org.apache.pig.backend.datastorage.DataStorage) + * @see org.apache.pig.LoadFunc#getLoadCaster() */ @Override - public Schema determineSchema(String fileName, ExecType execType, - DataStorage storage) throws IOException { + public LoadCaster getLoadCaster() throws IOException { // TODO Auto-generated method stub return null; } /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#fieldsToRead(org.apache.pig.impl.logicalLayer.schema.Schema) + * @see org.apache.pig.LoadFunc#prepareToRead(org.apache.hadoop.mapreduce.RecordReader, org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit) */ @Override - public void fieldsToRead(Schema schema) { + public void prepareToRead(RecordReader reader, PigSplit split) + throws IOException { // TODO Auto-generated method stub } /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#getNext() + * @see org.apache.pig.LoadFunc#relativeToAbsolutePath(java.lang.String, org.apache.hadoop.fs.Path) */ @Override - public Tuple getNext() throws IOException { + public String relativeToAbsolutePath(String location, Path curDir) + throws IOException { // TODO Auto-generated method stub return null; } /* (non-Javadoc) - * @see org.apache.pig.IndexableLoadFunc#initialize(org.apache.hadoop.conf.Configuration) + * @see org.apache.pig.LoadFunc#setLocation(java.lang.String, org.apache.hadoop.mapreduce.Job) */ @Override - public void initialize(Configuration conf) throws IOException { + public void setLocation(String location, Job job) throws IOException { // TODO Auto-generated method stub } Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestNullConstant.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestNullConstant.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestNullConstant.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestNullConstant.java Wed Nov 18 01:03:14 2009 @@ -36,9 +36,10 @@ @Test public void testArithExpressions() throws IOException, ExecException { - File input = Util.createInputFile("tmp", "", + String inputFileName = "testArithExpressions-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double);"); pigServer.registerQuery("b = foreach a generate x + null, x * null, x / null, x - null, null % x, " + "y + null, y * null, y / null, y - null;"); Iterator<Tuple> it = pigServer.openIterator("b"); @@ -46,13 +47,15 @@ for (int i = 0; i < 9; i++) { assertEquals(null, t.get(i)); } + Util.deleteFile(cluster, inputFileName); } @Test public void testBinCond() throws IOException, ExecException { - File input = Util.createInputFile("tmp", "", + String inputFileName = "testBinCond-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double);"); pigServer.registerQuery("b = foreach a generate (2 > 1? null : 1), ( 2 < 1 ? null : 1), (2 > 1 ? 1 : null), ( 2 < 1 ? 1 : null);"); Iterator<Tuple> it = pigServer.openIterator("b"); Tuple t = it.next(); @@ -68,14 +71,16 @@ for (int i = 0; i < 2; i++) { assertEquals(1, t.get(i)); } + Util.deleteFile(cluster, inputFileName); } @Test public void testForeachGenerate() throws ExecException, IOException { - File input = Util.createInputFile("tmp", "", + String inputFileName = "testForeachGenerate-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double);"); pigServer.registerQuery("b = foreach a generate x, null, y, null;"); Iterator<Tuple> it = pigServer.openIterator("b"); Tuple t = it.next(); @@ -83,17 +88,20 @@ for (int i = 0; i < 4; i++) { assertEquals(result[i], t.get(i)); } + Util.deleteFile(cluster, inputFileName); } @Test public void testOuterJoin() throws IOException, ExecException { - File input1 = Util.createInputFile("tmp", "", + String inputFileName1 = "testOuterJoin-input1.txt"; + Util.createInputFile(cluster, inputFileName1, new String[] {"10\twill_join", "11\twill_not_join"}); - File input2 = Util.createInputFile("tmp", "", + String inputFileName2 = "testOuterJoin-input2.txt"; + Util.createInputFile(cluster, inputFileName2, new String[] {"10\twill_join", "12\twill_not_join"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input1.toString()) + "' as (x:int, y:chararray);"); - pigServer.registerQuery("b = load 'file:" + Util.encodeEscape(input2.toString()) + "' as (u:int, v:chararray);"); + pigServer.registerQuery("a = load '" + inputFileName1 + "' as (x:int, y:chararray);"); + pigServer.registerQuery("b = load '" + inputFileName2 + "' as (u:int, v:chararray);"); pigServer.registerQuery("c = cogroup a by x, b by u;"); pigServer.registerQuery("d = foreach c generate flatten((SIZE(a) == 0 ? null : a)), flatten((SIZE(b) == 0 ? null : b));"); Iterator<Tuple> it = pigServer.openIterator("d"); @@ -108,13 +116,16 @@ assertEquals(result[j], t.get(j)); } } + Util.deleteFile(cluster, inputFileName1); + Util.deleteFile(cluster, inputFileName2); } @Test public void testConcatAndSize() throws IOException, ExecException { - File input = Util.createInputFile("tmp", "", + String inputFileName = "testConcatAndSize-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0\tstring"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double, str:chararray);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double, str:chararray);"); pigServer.registerQuery("b = foreach a generate SIZE(null), CONCAT(str, null), " + "CONCAT(null, str);"); Iterator<Tuple> it = pigServer.openIterator("b"); @@ -122,41 +133,45 @@ for (int i = 0; i < 3; i++) { assertEquals(null, t.get(i)); } + Util.deleteFile(cluster, inputFileName); } @Test public void testExplicitCast() throws IOException, ExecException { - File input = Util.createInputFile("tmp", "", + String inputFileName = "testExplicitCast-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0\tstring"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double, str:chararray);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double, str:chararray);"); pigServer.registerQuery("b = foreach a generate (int)null, (double)null, (chararray)null, (map[])null;"); Iterator<Tuple> it = pigServer.openIterator("b"); Tuple t = it.next(); for (int i = 0; i < 3; i++) { assertEquals(null, t.get(i)); } + Util.deleteFile(cluster, inputFileName); } @Test public void testComplexNullConstants() throws IOException, ExecException { - File input = Util.createInputFile("tmp", "", + String inputFileName = "testComplexNullConstants-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0\tstring"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double, str:chararray);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double, str:chararray);"); pigServer.registerQuery("b = foreach a generate {(null)}, ['2'#null];"); Iterator<Tuple> it = pigServer.openIterator("b"); Tuple t = it.next(); System.out.println("tuple: " + t); assertEquals(null, ((DataBag)t.get(0)).iterator().next().get(0)); assertEquals(null, ((Map<String, Object>)t.get(1)).get("2")); - + Util.deleteFile(cluster, inputFileName); } @Test public void testMapNullKeyFailure() throws IOException { - File input; - input = Util.createInputFile("tmp", "", + String inputFileName = "testMapNullKeyFailure-input.txt"; + Util.createInputFile(cluster, inputFileName, new String[] {"10\t11.0\tstring"}); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(input.toString()) + "' as (x:int, y:double, str:chararray);"); + pigServer.registerQuery("a = load '" + inputFileName + "' as (x:int, y:double, str:chararray);"); boolean exceptionOccured = false; try { @@ -166,6 +181,7 @@ String msg = e.getMessage(); assertTrue(msg.contains("key in a map cannot be null")); } + Util.deleteFile(cluster, inputFileName); if(!exceptionOccured) fail(); } } Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPOCast.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPOCast.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPOCast.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPOCast.java Wed Nov 18 01:03:14 2009 @@ -24,8 +24,13 @@ import java.util.Map; import java.util.Random; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordReader; import org.apache.pig.ExecType; import org.apache.pig.FuncSpec; +import org.apache.pig.LoadCaster; import org.apache.pig.LoadFunc; import org.apache.pig.backend.datastorage.DataStorage; import org.apache.pig.backend.executionengine.ExecException; @@ -39,6 +44,7 @@ import org.apache.pig.impl.io.BufferedPositionedInputStream; import org.apache.pig.impl.plan.OperatorKey; import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; @@ -61,7 +67,7 @@ DataBag dummyBag = null; @Test - public void testIntegerToOther() throws PlanException, ExecException { + public void testIntegerToOther() throws IOException { //Create data DataBag bag = BagFactory.getInstance().newDefaultBag(); for(int i = 0; i < MAX; i++) { @@ -224,7 +230,7 @@ } @Test - public void testLongToOther() throws PlanException, ExecException { + public void testLongToOther() throws IOException { //Create data DataBag bag = BagFactory.getInstance().newDefaultBag(); for(int i = 0; i < MAX; i++) { @@ -395,7 +401,7 @@ } @Test - public void testFloatToOther() throws PlanException, ExecException { + public void testFloatToOther() throws IOException { //Create data DataBag bag = BagFactory.getInstance().newDefaultBag(); for(int i = 0; i < MAX; i++) { @@ -570,7 +576,7 @@ } @Test - public void testDoubleToOther() throws PlanException, ExecException { + public void testDoubleToOther() throws IOException { //Create data DataBag bag = BagFactory.getInstance().newDefaultBag(); for(int i = 0; i < MAX; i++) { @@ -734,7 +740,7 @@ } @Test - public void testStringToOther() throws PlanException, ExecException { + public void testStringToOther() throws IOException { POCast op = new POCast(new OperatorKey("", r.nextLong()), -1); LoadFunc load = new TestLoader(); op.setLoadFSpec(new FuncSpec(load.getClass().getName())); @@ -901,7 +907,7 @@ } } - public static class TestLoader implements LoadFunc{ + public static class TestLoader implements LoadFunc, LoadCaster{ public void bindTo(String fileName, BufferedPositionedInputStream is, long offset, long end) throws IOException { } @@ -986,18 +992,57 @@ public byte[] toBytes(Tuple t) throws IOException { return null; } + + /* (non-Javadoc) + * @see org.apache.pig.LoadFunc#getInputFormat() + */ + @Override + public InputFormat getInputFormat() throws IOException { + // TODO Auto-generated method stub + return null; + } + + /* (non-Javadoc) + * @see org.apache.pig.LoadFunc#getLoadCaster() + */ + @Override + public LoadCaster getLoadCaster() throws IOException { + return this; + } + + /* (non-Javadoc) + * @see org.apache.pig.LoadFunc#prepareToRead(org.apache.hadoop.mapreduce.RecordReader, org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit) + */ + @Override + public void prepareToRead(RecordReader reader, PigSplit split) + throws IOException { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) - * @see org.apache.pig.LoadFunc#determineSchema(java.lang.String, org.apache.pig.ExecType, org.apache.pig.backend.datastorage.DataStorage) + * @see org.apache.pig.LoadFunc#relativeToAbsolutePath(java.lang.String, org.apache.hadoop.fs.Path) */ - public Schema determineSchema(String fileName, ExecType execType, - DataStorage storage) throws IOException { + @Override + public String relativeToAbsolutePath(String location, Path curDir) + throws IOException { // TODO Auto-generated method stub return null; } + + /* (non-Javadoc) + * @see org.apache.pig.LoadFunc#setLocation(java.lang.String, org.apache.hadoop.mapreduce.Job) + */ + @Override + public void setLocation(String location, Job job) throws IOException { + // TODO Auto-generated method stub + + } + } @Test - public void testByteArrayToOther() throws PlanException, ExecException { + public void testByteArrayToOther() throws IOException { POCast op = new POCast(new OperatorKey("", r.nextLong()), -1); LoadFunc load = new TestLoader(); op.setLoadFSpec(new FuncSpec(load.getClass().getName())); @@ -1160,7 +1205,7 @@ } } - private PhysicalPlan constructPlan(POCast op) throws PlanException { + private PhysicalPlan constructPlan(POCast op) throws IOException { LoadFunc load = new TestLoader(); op.setLoadFSpec(new FuncSpec(load.getClass().getName())); POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0); @@ -1186,7 +1231,7 @@ * the input to the cast is already a string */ @Test - public void testByteArrayToOtherNoCast() throws PlanException, ExecException { + public void testByteArrayToOtherNoCast() throws IOException { POCast op = new POCast(new OperatorKey("", r.nextLong()), -1); PhysicalPlan plan = constructPlan(op); TupleFactory tf = TupleFactory.getInstance(); @@ -1310,7 +1355,7 @@ } @Test - public void testTupleToOther() throws PlanException, ExecException { + public void testTupleToOther() throws IOException { POCast op = new POCast(new OperatorKey("", r.nextLong()), -1); op.setLoadFSpec(new FuncSpec(PigStorage.class.getName())); POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0); @@ -1438,7 +1483,7 @@ } @Test - public void testBagToOther() throws PlanException, ExecException { + public void testBagToOther() throws IOException { POCast op = new POCast(new OperatorKey("", r.nextLong()), -1); op.setLoadFSpec(new FuncSpec(PigStorage.class.getName())); POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0); @@ -1547,7 +1592,7 @@ } @Test - public void testMapToOther() throws PlanException, ExecException { + public void testMapToOther() throws IOException { POCast op = new POCast(new OperatorKey("", r.nextLong()), -1); op.setLoadFSpec(new FuncSpec(PigStorage.class.getName())); POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0); @@ -1748,7 +1793,7 @@ } @Test - public void testValueTypesChanged() throws PlanException, ExecException { + public void testValueTypesChanged() throws IOException { // Plan to test when result type is ByteArray and casting is requested // for example casting of values coming out of map lookup. Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPigSplit.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPigSplit.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPigSplit.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPigSplit.java Wed Nov 18 01:03:14 2009 @@ -19,23 +19,57 @@ package org.apache.pig.test; import java.io.File; +import java.io.IOException; import java.io.PrintWriter; import java.util.Iterator; +import org.apache.pig.ExecType; import org.apache.pig.data.Tuple; import org.junit.Test; public class TestPigSplit extends PigExecTestCase { - + + /** + * filename of input in each of the tests + */ + String inputFileName; + + public TestPigSplit() { + // by default let's test in map-red mode, user can + // override this by setting ant property "test.exectype" + execType = ExecType.MAPREDUCE; + } + + private void createInput(String[] data) throws IOException { + if(execType == ExecType.MAPREDUCE) { + Util.createInputFile(cluster, inputFileName, data); + } else if (execType == ExecType.LOCAL) { + Util.createLocalInputFile(inputFileName, data); + } else { + throw new IOException("unknown exectype:" + execType.toString()); + } + } + + /* (non-Javadoc) + * @see org.apache.pig.test.PigExecTestCase#tearDown() + */ + @Override + protected void tearDown() throws Exception { + if(execType == ExecType.MAPREDUCE) { + Util.deleteFile(cluster, inputFileName); + } else if (execType == ExecType.LOCAL) { + new File(inputFileName).delete(); + } else { + throw new IOException("unknown exectype:" + execType.toString()); + } + } + @Test public void notestLongEvalSpec() throws Exception{ - File f = File.createTempFile("tmp", ""); + inputFileName = "notestLongEvalSpec-input.txt"; + createInput(new String[] {"0\ta"}); - PrintWriter pw = new PrintWriter(f); - pw.println("0\ta"); - pw.close(); - - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(f.toString()) + "';"); + pigServer.registerQuery("a = load '" + inputFileName + "';"); for (int i=0; i< 500; i++){ pigServer.registerQuery("a = filter a by $0 == '1';"); } @@ -43,19 +77,14 @@ while (iter.hasNext()){ throw new Exception(); } - f.delete(); } @Test public void testSchemaWithSplit() throws Exception { - File f = File.createTempFile("tmp", ""); - - PrintWriter pw = new PrintWriter(f); - pw.println("2"); - pw.println("12"); - pw.println("42"); - pw.close(); - pigServer.registerQuery("a = load 'file:" + Util.encodeEscape(f.toString()) + "' as (value:chararray);"); + inputFileName = "testSchemaWithSplit-input.txt"; + String[] input = {"2","12","42"}; + createInput(input); + pigServer.registerQuery("a = load '" + inputFileName + "' as (value:chararray);"); pigServer.registerQuery("split a into b if value < '20', c if value > '10';"); pigServer.registerQuery("b1 = order b by value;"); pigServer.registerQuery("c1 = order c by value;"); @@ -77,26 +106,22 @@ assertEquals("third item in c1", iter.next().get(0), "42"); assertFalse("c1 is over", iter.hasNext()); - f.delete(); } @Test public void testLongEvalSpec() throws Exception{ - File f = File.createTempFile("tmp", ""); - - PrintWriter pw = new PrintWriter(f); + inputFileName = "testLongEvalSpec-input.txt"; + String[] input = new String[500]; for (int i=0; i< 500; i++) { - pw.println("0\ta"); + input[i] = ("0\ta"); } - pw.close(); - - pigServer.registerQuery("a = load '" + Util.generateURI(f.toString()) + "';"); + createInput(input); + pigServer.registerQuery("a = load '" + inputFileName + "';"); pigServer.registerQuery("a = filter a by $0 == '1';"); Iterator<Tuple> iter = pigServer.openIterator("a"); while (iter.hasNext()){ throw new Exception(); } - f.delete(); } } Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSample.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSample.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSample.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSample.java Wed Nov 18 01:03:14 2009 @@ -40,28 +40,28 @@ private String tmpfilepath; private int DATALEN = 1024; - + MiniCluster cluster = MiniCluster.buildCluster(); + @Override protected void setUp() throws Exception { - //pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); - pig = new PigServer("local"); + pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); tmpFile = File.createTempFile( this.getName(), ".txt"); - PrintStream ps = new PrintStream(new FileOutputStream(tmpFile)); + String input[] = new String[DATALEN]; for(int i = 0; i < DATALEN; i++) { - ps.println(i); + input[i] = Integer.toString(i); } - ps.close(); - - tmpfilepath = Util.generateURI(tmpFile.getCanonicalPath()); + + tmpfilepath = tmpFile.getCanonicalPath(); + Util.createInputFile(cluster, tmpfilepath, input); } protected void tearDown() throws Exception { - tmpFile.delete(); + Util.deleteFile(cluster, tmpfilepath); } private void verify(String query, int expected_min, int expected_max) Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStore.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStore.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStore.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStore.java Wed Nov 18 01:03:14 2009 @@ -161,15 +161,15 @@ for(String line=br.readLine();line!=null;line=br.readLine()){ String[] flds = line.split(":",-1); t = new DefaultTuple(); - t.append(flds[0].compareTo("")!=0 ? ps.bytesToBag(flds[0].getBytes()) : null); - t.append(flds[1].compareTo("")!=0 ? ps.bytesToCharArray(flds[1].getBytes()) : null); - t.append(flds[2].compareTo("")!=0 ? ps.bytesToCharArray(flds[2].getBytes()) : null); - t.append(flds[3].compareTo("")!=0 ? ps.bytesToDouble(flds[3].getBytes()) : null); - t.append(flds[4].compareTo("")!=0 ? ps.bytesToFloat(flds[4].getBytes()) : null); - t.append(flds[5].compareTo("")!=0 ? ps.bytesToInteger(flds[5].getBytes()) : null); - t.append(flds[6].compareTo("")!=0 ? ps.bytesToLong(flds[6].getBytes()) : null); - t.append(flds[7].compareTo("")!=0 ? ps.bytesToMap(flds[7].getBytes()) : null); - t.append(flds[8].compareTo("")!=0 ? ps.bytesToTuple(flds[8].getBytes()) : null); + t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes()) : null); + t.append(flds[1].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[1].getBytes()) : null); + t.append(flds[2].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[2].getBytes()) : null); + t.append(flds[3].compareTo("")!=0 ? ps.getLoadCaster().bytesToDouble(flds[3].getBytes()) : null); + t.append(flds[4].compareTo("")!=0 ? ps.getLoadCaster().bytesToFloat(flds[4].getBytes()) : null); + t.append(flds[5].compareTo("")!=0 ? ps.getLoadCaster().bytesToInteger(flds[5].getBytes()) : null); + t.append(flds[6].compareTo("")!=0 ? ps.getLoadCaster().bytesToLong(flds[6].getBytes()) : null); + t.append(flds[7].compareTo("")!=0 ? ps.getLoadCaster().bytesToMap(flds[7].getBytes()) : null); + t.append(flds[8].compareTo("")!=0 ? ps.getLoadCaster().bytesToTuple(flds[8].getBytes()) : null); assertEquals(true, TestHelper.bagContains(inpDB, t)); ++size; @@ -196,16 +196,16 @@ System.err.println(line); String[] flds = line.split(":",-1); t = new DefaultTuple(); - t.append(flds[0].compareTo("")!=0 ? ps.bytesToBag(flds[0].getBytes()) : null); - t.append(flds[1].compareTo("")!=0 ? ps.bytesToCharArray(flds[1].getBytes()) : null); - t.append(flds[2].compareTo("")!=0 ? ps.bytesToCharArray(flds[2].getBytes()) : null); - t.append(flds[3].compareTo("")!=0 ? ps.bytesToDouble(flds[3].getBytes()) : null); - t.append(flds[4].compareTo("")!=0 ? ps.bytesToFloat(flds[4].getBytes()) : null); - t.append(flds[5].compareTo("")!=0 ? ps.bytesToInteger(flds[5].getBytes()) : null); - t.append(flds[6].compareTo("")!=0 ? ps.bytesToLong(flds[6].getBytes()) : null); - t.append(flds[7].compareTo("")!=0 ? ps.bytesToMap(flds[7].getBytes()) : null); - t.append(flds[8].compareTo("")!=0 ? ps.bytesToTuple(flds[8].getBytes()) : null); - t.append(flds[9].compareTo("")!=0 ? ps.bytesToCharArray(flds[9].getBytes()) : null); + t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes()) : null); + t.append(flds[1].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[1].getBytes()) : null); + t.append(flds[2].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[2].getBytes()) : null); + t.append(flds[3].compareTo("")!=0 ? ps.getLoadCaster().bytesToDouble(flds[3].getBytes()) : null); + t.append(flds[4].compareTo("")!=0 ? ps.getLoadCaster().bytesToFloat(flds[4].getBytes()) : null); + t.append(flds[5].compareTo("")!=0 ? ps.getLoadCaster().bytesToInteger(flds[5].getBytes()) : null); + t.append(flds[6].compareTo("")!=0 ? ps.getLoadCaster().bytesToLong(flds[6].getBytes()) : null); + t.append(flds[7].compareTo("")!=0 ? ps.getLoadCaster().bytesToMap(flds[7].getBytes()) : null); + t.append(flds[8].compareTo("")!=0 ? ps.getLoadCaster().bytesToTuple(flds[8].getBytes()) : null); + t.append(flds[9].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[9].getBytes()) : null); assertTrue(TestHelper.tupleEquals(inputTuple, t)); ++size; Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java?rev=881640&r1=881639&r2=881640&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java Wed Nov 18 01:03:14 2009 @@ -23,6 +23,7 @@ import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; @@ -31,6 +32,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -208,14 +210,28 @@ throws IOException { File f = File.createTempFile(tmpFilenamePrefix, tmpFilenameSuffix); f.deleteOnExit(); - PrintWriter pw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8")); - for (int i=0; i<inputData.length; i++){ - pw.println(inputData[i]); - } - pw.close(); + writeToFile(f, inputData); return f; } + static public File createLocalInputFile(String filename, String[] inputData) + throws IOException { + File f = new File(filename); + f.deleteOnExit(); + writeToFile(f, inputData); + return f; + } + + private static void writeToFile(File f, String[] inputData) throws + IOException { + PrintWriter pw = new PrintWriter(new OutputStreamWriter(new + FileOutputStream(f), "UTF-8")); + for (int i=0; i<inputData.length; i++){ + pw.println(inputData[i]); + } + pw.close(); + } + /** * Helper to create a dfs file on the Minicluster DFS with given * input data for use in test cases.
