Author: pradeepkth
Date: Wed Dec 23 22:02:49 2009
New Revision: 893637
URL: http://svn.apache.org/viewvc?rev=893637&view=rev
Log:
PIG-1148: Move splitable logic from pig latin to InputFormat (zjffdu via
pradeepkth)
Modified:
hadoop/pig/branches/load-store-redesign/CHANGES.txt
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SampleOptimizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLoad.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestFRJoin.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestInputOutputFileValidator.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLocalJobSubmission.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TestHelper.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/CHANGES.txt?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/CHANGES.txt Wed Dec 23 22:02:49 2009
@@ -22,6 +22,9 @@
INCOMPATIBLE CHANGES
+PIG-1148: Move splitable logic from pig latin to InputFormat (zjffdu via
+pradeepkth)
+
PIG-1141: Make streaming work with the new load-store interfaces (rding via
pradeepkth)
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Wed Dec 23 22:02:49 2009
@@ -341,7 +341,7 @@
Configuration conf = nwJob.getConfiguration();
- ArrayList<Pair<FileSpec, Boolean>> inp = new ArrayList<Pair<FileSpec,
Boolean>>();
+ ArrayList<FileSpec> inp = new ArrayList<FileSpec>();
ArrayList<List<OperatorKey>> inpTargets = new
ArrayList<List<OperatorKey>>();
ArrayList<String> inpSignatureLists = new ArrayList<String>();
ArrayList<POStore> storeLocations = new ArrayList<POStore>();
@@ -372,9 +372,8 @@
if(lds!=null && lds.size()>0){
for (POLoad ld : lds) {
- Pair<FileSpec, Boolean> p = new Pair<FileSpec,
Boolean>(ld.getLFile(), ld.isSplittable());
//Store the inp filespecs
- inp.add(p);
+ inp.add(ld.getLFile());
//Store the target operators for tuples read
//from this input
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Wed Dec 23 22:02:49 2009
@@ -379,7 +379,7 @@
}
private POLoad getLoad(){
- POLoad ld = new POLoad(new
OperatorKey(scope,nig.getNextNodeId(scope)), true);
+ POLoad ld = new POLoad(new
OperatorKey(scope,nig.getNextNodeId(scope)));
ld.setPc(pigContext);
return ld;
}
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
Wed Dec 23 22:02:49 2009
@@ -127,19 +127,19 @@
*/
@SuppressWarnings("unchecked")
private static LoadFunc getLoadFunc(int inputIndex, Configuration conf)
throws IOException {
- ArrayList<Pair<FileSpec, Boolean>> inputs =
- (ArrayList<Pair<FileSpec, Boolean>>) ObjectSerializer.deserialize(
+ ArrayList<FileSpec> inputs =
+ (ArrayList<FileSpec>) ObjectSerializer.deserialize(
conf.get(PIG_INPUTS));
- FuncSpec loadFuncSpec = inputs.get(inputIndex).first.getFuncSpec();
+ FuncSpec loadFuncSpec = inputs.get(inputIndex).getFuncSpec();
return (LoadFunc) PigContext.instantiateFuncFromSpec(loadFuncSpec);
}
@SuppressWarnings("unchecked")
private static String getLoadLocation(int inputIndex, Configuration conf)
throws IOException {
- ArrayList<Pair<FileSpec, Boolean>> inputs =
- (ArrayList<Pair<FileSpec, Boolean>>) ObjectSerializer.deserialize(
+ ArrayList<FileSpec> inputs =
+ (ArrayList<FileSpec>) ObjectSerializer.deserialize(
conf.get(PIG_INPUTS));
- return inputs.get(inputIndex).first.getFileName();
+ return inputs.get(inputIndex).getFileName();
}
/**
@@ -182,11 +182,11 @@
Configuration conf = jobcontext.getConfiguration();
- ArrayList<Pair<FileSpec, Boolean>> inputs;
+ ArrayList<FileSpec> inputs;
ArrayList<ArrayList<OperatorKey>> inpTargets;
PigContext pigContext;
try {
- inputs = (ArrayList<Pair<FileSpec, Boolean>>) ObjectSerializer
+ inputs = (ArrayList<FileSpec>) ObjectSerializer
.deserialize(conf.get("pig.inputs"));
inpTargets = (ArrayList<ArrayList<OperatorKey>>) ObjectSerializer
.deserialize(conf.get("pig.inpTargets"));
@@ -202,7 +202,7 @@
ArrayList<InputSplit> splits = new ArrayList<InputSplit>();
for (int i = 0; i < inputs.size(); i++) {
try {
- Path path = new Path(inputs.get(i).first.getFileName());
+ Path path = new Path(inputs.get(i).getFileName());
FileSystem fs;
@@ -230,7 +230,7 @@
// FileInputFormat stores this in mapred.input.dir in the
conf),
// then for different inputs, the loader's don't end up
// over-writing the same conf.
- FuncSpec loadFuncSpec = inputs.get(i).first.getFuncSpec();
+ FuncSpec loadFuncSpec = inputs.get(i).getFuncSpec();
LoadFunc loadFunc = (LoadFunc)
PigContext.instantiateFuncFromSpec(
loadFuncSpec);
Configuration confClone = new Configuration(conf);
@@ -239,7 +239,7 @@
// the conf
passLoadSignature(loadFunc, i,
inputSpecificJob.getConfiguration(), false);
- loadFunc.setLocation(inputs.get(i).first.getFileName(),
+ loadFunc.setLocation(inputs.get(i).getFileName(),
inputSpecificJob);
// The above setLocation call could write to the conf within
// the inputSpecificJob - use this updated conf
@@ -256,7 +256,7 @@
throw ee;
} catch (Exception e) {
int errCode = 2118;
- String msg = "Unable to create input splits for: " +
inputs.get(i).first.getFileName();
+ String msg = "Unable to create input splits for: " +
inputs.get(i).getFileName();
throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SampleOptimizer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SampleOptimizer.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SampleOptimizer.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SampleOptimizer.java
Wed Dec 23 22:02:49 2009
@@ -173,7 +173,7 @@
// Second argument is the number of samples per block, read this from
the original.
rslargs[1] = load.getLFile().getFuncSpec().getCtorArgs()[1];
FileSpec fs = new FileSpec(predFs.getFileName(),new FuncSpec(loadFunc,
rslargs));
- POLoad newLoad = new
POLoad(load.getOperatorKey(),load.getRequestedParallelism(), fs,
load.isSplittable());
+ POLoad newLoad = new
POLoad(load.getOperatorKey(),load.getRequestedParallelism(), fs);
newLoad.setSignature(load.getSignature());
try {
mr.mapPlan.replace(load, newLoad);
@@ -189,7 +189,7 @@
// Second, replace the loader in our successor with whatever the
originally used loader was.
fs = new FileSpec(predFs.getFileName(), predFs.getFuncSpec());
- newLoad = new POLoad(succLoad.getOperatorKey(),
succLoad.getRequestedParallelism(), fs, succLoad.isSplittable());
+ newLoad = new POLoad(succLoad.getOperatorKey(),
succLoad.getRequestedParallelism(), fs);
newLoad.setSignature(succLoad.getSignature());
try {
succ.mapPlan.replace(succLoad, newLoad);
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
Wed Dec 23 22:02:49 2009
@@ -1567,7 +1567,7 @@
public void visit(LOLoad loLoad) throws VisitorException {
String scope = loLoad.getOperatorKey().scope;
POLoad load = new POLoad(new OperatorKey(scope, nodeGen
- .getNextNodeId(scope)), loLoad.isSplittable());
+ .getNextNodeId(scope)));
load.setAlias(loLoad.getAlias());
load.setLFile(loLoad.getInputFile());
load.setPc(pc);
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
Wed Dec 23 22:02:49 2009
@@ -310,7 +310,7 @@
}
POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L),
- replFile, false);
+ replFile);
PigContext pc = new PigContext(ExecType.MAPREDUCE,
ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
pc.connect();
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
Wed Dec 23 22:02:49 2009
@@ -63,8 +63,6 @@
PigContext pc;
//Indicates whether the loader setup is done or not
boolean setUpDone = false;
- // Indicates whether the filespec is splittable
- boolean splittable = true;
// default offset.
private long offset = 0;
// Alias for the POLoad
@@ -72,24 +70,23 @@
transient private final Log log = LogFactory.getLog(getClass());
- public POLoad(OperatorKey k, boolean splittable) {
- this(k,-1, null, splittable);
+ public POLoad(OperatorKey k) {
+ this(k,-1, null);
}
- public POLoad(OperatorKey k, FileSpec lFile, boolean splittable){
- this(k,-1,lFile, splittable);
+ public POLoad(OperatorKey k, FileSpec lFile){
+ this(k,-1,lFile);
}
- public POLoad(OperatorKey k, FileSpec lFile, long offset, boolean
splittable){
- this(k,-1,lFile, splittable);
+ public POLoad(OperatorKey k, FileSpec lFile, long offset){
+ this(k,-1,lFile);
this.offset = offset;
}
- public POLoad(OperatorKey k, int rp, FileSpec lFile,boolean splittable) {
+ public POLoad(OperatorKey k, int rp, FileSpec lFile) {
super(k, rp);
this.lFile = lFile;
- this.splittable = splittable;
}
/**
@@ -206,14 +203,6 @@
this.pc = pc;
}
-
- /**
- * @return the splittable
- */
- public boolean isSplittable() {
- return splittable;
- }
-
public String getSignature() {
return signature;
}
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
Wed Dec 23 22:02:49 2009
@@ -104,7 +104,7 @@
// there are multiple Join keys, the tuple itself represents
// the join key
Object firstLeftKey = (keys.size() == 1 ? keys.get(0): keys);
- POLoad ld = new POLoad(genKey(), new FileSpec(indexFile, new
FuncSpec(indexFileLoadFuncSpec)), false);
+ POLoad ld = new POLoad(genKey(), new FileSpec(indexFile, new
FuncSpec(indexFileLoadFuncSpec)));
try {
pc =
(PigContext)ObjectSerializer.deserialize(PigMapReduce.sJobConf.get("pig.pigContext"));
} catch (IOException e) {
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLoad.java
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLoad.java
Wed Dec 23 22:02:49 2009
@@ -59,7 +59,6 @@
public class LOLoad extends RelationalOperator {
private static final long serialVersionUID = 2L;
- protected boolean splittable = true;
private FileSpec mInputFileSpec;
transient private LoadFunc mLoadFunc;
@@ -82,13 +81,11 @@
* the execution mode @see org.apache.pig.ExecType
* @param storage
* the underlying storage
- * @param splittable
- * if the input file is splittable (.gz is not)
*
*
*/
public LOLoad(LogicalPlan plan, OperatorKey key, FileSpec inputFileSpec,
- ExecType execType, DataStorage storage, boolean splittable) throws
IOException {
+ ExecType execType, DataStorage storage) throws IOException {
super(plan, key);
mInputFileSpec = inputFileSpec;
//mSchemaFile = schemaFile;
@@ -97,7 +94,6 @@
mSchemaFile = inputFileSpec.getFileName();
mStorage = storage;
mExecType = execType;
- this.splittable = splittable;
// Generate a psudo alias. Since in the following script, we do not
have alias for LOLoad, however, alias is required.
// a = foreach (load '1') generate b0;
this.mAlias = ""+key.getId();
@@ -247,10 +243,6 @@
this.mEnforcedSchema = enforcedSchema;
}
- public boolean isSplittable() {
- return splittable;
- }
-
@Override
public byte getType() {
return DataType.BAG ;
Modified:
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Wed Dec 23 22:02:49 2009
@@ -1237,15 +1237,6 @@
(
<USING> funcSpec = NonEvalFuncSpec(FunctionType.LOADFUNC)
)?
- (
- <SPLIT> <BY> t3 = <QUOTEDSTRING>
- {
- splitBy = unquote(t3.image);
- if (splitBy.equalsIgnoreCase("file")) {
- splittable = false;
- }
- }
- )?
)
{
if (funcSpec == null){
@@ -1282,7 +1273,7 @@
fileNameMap.put(filename, absolutePath);
}
lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new
FileSpec(absolutePath, funcSpec),
- pigContext.getExecType(), pigContext.getFs(), splittable);
+ pigContext.getExecType(), pigContext.getFs());
} catch (IOException ioe) {
// The autogenerated parser code only catches RuntimeException and
// ParseException as special Exceptions. All others are caught as
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestFRJoin.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestFRJoin.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestFRJoin.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestFRJoin.java
Wed Dec 23 22:02:49 2009
@@ -129,7 +129,7 @@
private void setUpHashTable() throws IOException {
FileSpec replFile = new FileSpec(repl,new
FuncSpec(PigStorage.class.getName()+"()"));
- POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L),
replFile, false);
+ POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L),
replFile);
PigContext pc = new
PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
try {
pc.connect();
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestInputOutputFileValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestInputOutputFileValidator.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestInputOutputFileValidator.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestInputOutputFileValidator.java
Wed Dec 23 22:02:49 2009
@@ -148,7 +148,7 @@
new FileSpec(inputFile, new
FuncSpec("org.apache.pig.builtin.PigStorage")) ;
FileSpec filespec2 =
new FileSpec(outputFile, new
FuncSpec("org.apache.pig.builtin.PigStorage"));
- LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null,
dfs, true) ;
+ LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null,
dfs) ;
LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
plan.add(load) ;
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLocalJobSubmission.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLocalJobSubmission.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLocalJobSubmission.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLocalJobSubmission.java
Wed Dec 23 22:02:49 2009
@@ -392,7 +392,7 @@
FileSpec LFSpec = new FileSpec(ldFile, new
FuncSpec(PigStorage.class.getName(), new String[]{","}));
FileSpec SFSpec = new FileSpec(stFile, new
FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
Wed Dec 23 22:02:49 2009
@@ -951,11 +951,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1075,11 +1075,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1247,7 +1247,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1315,7 +1315,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1383,7 +1383,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1453,7 +1453,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1533,7 +1533,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1631,7 +1631,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1726,7 +1726,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1852,7 +1852,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1954,11 +1954,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -2119,11 +2119,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -2254,11 +2254,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, null) ;
// schema for input#1
Schema inputSchema1 = null ;
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
Wed Dec 23 22:02:49 2009
@@ -445,7 +445,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, new HDataStorage(new Properties()),
true) ;
+ null, new HDataStorage(new Properties())) ;
// set schemas
load1.setEnforcedSchema(null) ;
@@ -509,7 +509,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, new HDataStorage(new Properties()),
true) ;
+ null, new HDataStorage(new Properties())) ;
// set schemas
load1.setEnforcedSchema(null) ;
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
Wed Dec 23 22:02:49 2009
@@ -570,7 +570,7 @@
public static POLoad topLoadOp() {
- POLoad ret = new POLoad(new OperatorKey("", r.nextLong()), true);
+ POLoad ret = new POLoad(new OperatorKey("", r.nextLong()));
ret.setPc(pc);
return ret;
}
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TestHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TestHelper.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TestHelper.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TestHelper.java
Wed Dec 23 22:02:49 2009
@@ -171,11 +171,11 @@
public static boolean areFilesSame(FileSpec expLocal, FileSpec actHadoop,
PigContext pc) throws ExecException, IOException{
Random r = new Random();
- POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()), true);
+ POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()));
ldExp.setPc(pc);
ldExp.setLFile(expLocal);
- POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()), true);
+ POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()));
ldAct.setPc(pc);
ldAct.setLFile(actHadoop);
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
Wed Dec 23 22:02:49 2009
@@ -47,7 +47,7 @@
LOLoad load = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new
FuncSpec(pigStorage)),
- null, dfs, true) ;
+ null, dfs) ;
return load ;
} catch (IOException e) {
throw new AssertionError("This cannot happen") ;
Modified:
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=893637&r1=893636&r2=893637&view=diff
==============================================================================
---
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
(original)
+++
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
Wed Dec 23 22:02:49 2009
@@ -95,7 +95,7 @@
FileSpec fileSpec = new FileSpec("pi",
new
FuncSpec(PigStorage.class.getName())) ;
try {
- load = new LOLoad(plan, getKey(node.attributes), fileSpec, null,
null, true) ;
+ load = new LOLoad(plan, getKey(node.attributes), fileSpec, null,
null) ;
fillSchema(load, node.attributes) ;
}
catch (IOException ioe) {
