Author: pradeepkth
Date: Wed Dec 23 19:30:03 2009
New Revision: 893609
URL: http://svn.apache.org/viewvc?rev=893609&view=rev
Log:
PIG-1090: Update sources to reflect recent changes in load-store interfaces -
Removing dead code with new changes per load store redesign (rding via
pradeepkth)
Removed:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/executionengine/PigSlice.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/executionengine/PigSlicer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigDump.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/PigLineRecordReader.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/ValidatingInputFileSpec.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPigLineRecordReader.java
Modified:
hadoop/pig/branches/load-store-redesign/build.xml
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/FileLocalizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/streaming/OutputHandler.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreaming.java
Modified: hadoop/pig/branches/load-store-redesign/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/build.xml?rev=893609&r1=893608&r2=893609&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/build.xml (original)
+++ hadoop/pig/branches/load-store-redesign/build.xml Wed Dec 23 19:30:03 2009
@@ -296,13 +296,8 @@
<!-- This target is for default compilation -->
<target name="compile-sources" unless="all.warnings">
- <!-- FIXME: tests below are temporarily commented for load-store
redesign
- These should be fixed (removed from here) when the
src code
- is fixed for the corresponding feature
- FIXME: fix and remove when streaming and local mode is fixed -->
<javac encoding="${build.encoding}" srcdir="${sources}"
includes="**/*.java" destdir="${dist}" debug="${javac.debug}"
- excludes="**/TestPigLineRecordReader.java"
optimize="${javac.optimize}" target="${javac.version}"
source="${javac.version}" deprecation="${javac.deprecation}">
<compilerarg line="${javac.args} ${javac.args.warnings}"/>
@@ -551,14 +546,6 @@
<exclude name="**/TestOrderBy2.java" />
<exclude name="**/TestPi.java" />
<exclude name="**/nightly/**" />
-
- <!-- FIXME: tests below are temporarily commented for
load-store redesign
- These should be fixed (removed from here) when the
src code
- is fixed for the corresponding feature -->
- <!-- FIXME: fix and remove when streaming in fixed -->
- <!-- <exclude name="**/TestStreaming.java" /> -->
- <!-- FIXME: fix and remove when local mode is fixed -->
- <exclude name="**/TestPigLineRecordReader.java" />
</fileset>
</batchtest>
<batchtest fork="yes" todir="${test.log.dir}" if="testcase">
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=893609&r1=893608&r2=893609&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
Wed Dec 23 19:30:03 2009
@@ -38,15 +38,12 @@
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
import org.apache.pig.PigException;
-import org.apache.pig.backend.datastorage.DataStorage;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
-import org.apache.pig.impl.io.ValidatingInputFileSpec;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.Pair;
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/FileLocalizer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/FileLocalizer.java?rev=893609&r1=893608&r2=893609&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/FileLocalizer.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/FileLocalizer.java
Wed Dec 23 19:30:03 2009
@@ -44,9 +44,7 @@
import org.apache.pig.backend.datastorage.SeekableInputStream.FLAGS;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
-import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
-import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.WrappedIOException;
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=893609&r1=893608&r2=893609&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Wed Dec 23 19:30:03 2009
@@ -2346,7 +2346,7 @@
String stream, deserializer;
StreamingCommand.HandleSpec[] handleSpecs;
String functionName = PigStreaming.class.getName(), functionArgs="";
- byte funcType = (handle.compareTo(StreamingCommand.Handle.INPUT) != 0 ?
FunctionType.PIGTOSTREAMFUNC : FunctionType.STREAMTOPIGFUNC) ;
+ byte funcType = (handle.compareTo(StreamingCommand.Handle.INPUT) == 0 ?
FunctionType.PIGTOSTREAMFUNC : FunctionType.STREAMTOPIGFUNC) ;
FuncSpec funcSpec = null;
}
{
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/streaming/OutputHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/streaming/OutputHandler.java?rev=893609&r1=893608&r2=893609&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/streaming/OutputHandler.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/streaming/OutputHandler.java
Wed Dec 23 19:30:03 2009
@@ -20,9 +20,9 @@
import java.io.IOException;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.LineReader;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.BufferedPositionedInputStream;
-import org.apache.pig.impl.io.PigLineRecordReader;
/**
* {...@link OutputHandler} is responsible for handling the output of the
@@ -44,11 +44,9 @@
*/
protected StreamToPig deserializer;
- protected PigLineRecordReader in = null;
+ protected LineReader in = null;
private BufferedPositionedInputStream istream;
-
- private long end = Long.MAX_VALUE;
/**
* Get the handled <code>OutputType</code>.
@@ -70,15 +68,7 @@
public void bindTo(String fileName, BufferedPositionedInputStream is,
long offset, long end) throws IOException {
this.istream = is;
- this.in = new PigLineRecordReader(istream, offset, end);
- this.end = end;
-
- // Since we are not block aligned we throw away the first
- // record and cound on a different instance to read it
- if (offset != 0) {
- getNext();
- }
-
+ this.in = new LineReader(istream);
}
/**
@@ -88,13 +78,13 @@
* @throws IOException
*/
public Tuple getNext() throws IOException {
- if (in == null || in.getPosition() > end) {
+ if (in == null) {
return null;
}
Text value = new Text();
- boolean notDone = in.next(value);
- if (!notDone) {
+ int num = in.readLine(value);
+ if (num <= 0) {
return null;
}
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreaming.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreaming.java?rev=893609&r1=893608&r2=893609&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreaming.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreaming.java
Wed Dec 23 19:30:03 2009
@@ -18,6 +18,7 @@
package org.apache.pig.test;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -31,6 +32,8 @@
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.streaming.PigStreaming;
+import org.apache.pig.impl.streaming.PigToStream;
+import org.apache.pig.impl.util.TupleFormat;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -723,7 +726,7 @@
// Pig query to run
pigServer.registerQuery("define CMD `"+ simpleEchoStreamingCommand
+
- "` input(stdin);");
+ "` input(stdin using " +
PigStreamDump.class.getName() + ");");
pigServer.registerQuery("IP = load '"
+ Util.generateURI(Util.encodeEscape(input.toString()),
pigServer.getPigContext()) + "' using "
@@ -741,4 +744,15 @@
Util.checkQueryOutputs(pigServer.openIterator("OP"),
expectedResults);
}
}
+
+ public static class PigStreamDump implements PigToStream {
+
+ public static final String recordDelimiter = "\n";
+
+ @Override
+ public byte[] serialize(Tuple t) throws IOException {
+ return (TupleFormat.format(t) + recordDelimiter).getBytes();
+ }
+
+ }
}
