Author: pradeepkth
Date: Wed Jan 6 17:27:17 2010
New Revision: 896552
URL: http://svn.apache.org/viewvc?rev=896552&view=rev
Log:
This is to cleanup the local mode code after switching to using hadoop local
mode
Added:
hadoop/pig/trunk/src/org/apache/pig/pen/LocalLogToPhyTranslationVisitor.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/LocalLogToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCogroup.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCounter.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/counters/POCounter.java
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCross.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCross.java
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplit.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplit.java
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplitOutput.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplitOutput.java
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POStreamLocal.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POStreamLocal.java
hadoop/pig/trunk/test/org/apache/pig/test/utils/LocalSeekableInputStream.java
- copied, changed from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalSeekableInputStream.java
Removed:
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalDataStorage.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalDir.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalFile.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalPath.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalSeekableInputStream.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalJob.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPOStoreImpl.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalResult.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/LocalLogToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/counters/POCounter.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCross.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplit.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplitOutput.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POStreamLocal.java
hadoop/pig/trunk/test/org/apache/pig/test/TestLocalJobSubmission.java
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/JsonMetadata.java
hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java
hadoop/pig/trunk/src/org/apache/pig/pen/DerivedDataVisitor.java
hadoop/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java
hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java
hadoop/pig/trunk/test/org/apache/pig/test/TestLocalPOSplit.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPOCogroup.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPOCross.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigLineRecordReader.java
hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
Wed Jan 6 17:27:17 2010
@@ -21,7 +21,7 @@
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.*;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.*;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.*;
+import org.apache.pig.pen.physicalOperators.*;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.impl.plan.DependencyOrderWalker;
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
Wed Jan 6 17:27:17 2010
@@ -23,7 +23,7 @@
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.*;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.*;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.*;
+import org.apache.pig.pen.physicalOperators.*;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.PlanWalker;
import org.apache.pig.impl.plan.VisitorException;
@@ -270,7 +270,7 @@
}
- public void
visitSplit(org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit
split) {
+ public void visitSplit(org.apache.pig.pen.physicalOperators.POSplit split)
{
// TODO Auto-generated method stub
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/JsonMetadata.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/JsonMetadata.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/JsonMetadata.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/JsonMetadata.java Wed Jan
6 17:27:17 2010
@@ -34,9 +34,6 @@
import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.backend.hadoop.datastorage.HDirectory;
import org.apache.pig.backend.hadoop.datastorage.HFile;
-import org.apache.pig.backend.local.datastorage.LocalDataStorage;
-import org.apache.pig.backend.local.datastorage.LocalDir;
-import org.apache.pig.backend.local.datastorage.LocalFile;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.plan.Operator;
@@ -115,11 +112,6 @@
Path parent = descriptorPath.getParent();
parentName = parent.toString();
parentContainer = new
HDirectory((HDataStorage)storage,parent);
- } else if (descriptor instanceof LocalFile) {
- File descriptorPath = ((LocalFile) descriptor).getPath();
- fileName = descriptorPath.getName();
- parentName = descriptorPath.getParent();
- parentContainer = new
LocalDir((LocalDataStorage)storage,parentName);
}
ElementDescriptor metaFilePath = storage.asElement(parentName,
prefix+"."+fileName);
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java Wed Jan 6
17:27:17 2010
@@ -54,9 +54,6 @@
import org.apache.pig.backend.executionengine.ExecutionEngine;
import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
-//import
org.apache.pig.backend.hadoop.executionengine.mapreduceExec.MapReduceLauncher;
-//import
org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigMapReduce;
-import org.apache.pig.backend.local.executionengine.LocalExecutionEngine;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.backend.hadoop.streaming.HadoopExecutableManager;
import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
Modified: hadoop/pig/trunk/src/org/apache/pig/pen/DerivedDataVisitor.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/DerivedDataVisitor.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/pen/DerivedDataVisitor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/pen/DerivedDataVisitor.java Wed Jan 6
17:27:17 2010
@@ -35,7 +35,6 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORead;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.LocalLogToPhyTranslationVisitor;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
Modified: hadoop/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java Wed Jan 6
17:27:17 2010
@@ -31,7 +31,6 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.LocalLogToPhyTranslationVisitor;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
Copied:
hadoop/pig/trunk/src/org/apache/pig/pen/LocalLogToPhyTranslationVisitor.java
(from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/LocalLogToPhyTranslationVisitor.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/LocalLogToPhyTranslationVisitor.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/LocalLogToPhyTranslationVisitor.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/LocalLogToPhyTranslationVisitor.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/LocalLogToPhyTranslationVisitor.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/pen/LocalLogToPhyTranslationVisitor.java
Wed Jan 6 17:27:17 2010
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.local.executionengine.physicalLayer;
+package org.apache.pig.pen;
import java.util.ArrayList;
import java.util.List;
@@ -33,12 +33,12 @@
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrangeForIllustrate;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.counters.POCounter;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POCogroup;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POCross;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplitOutput;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POStreamLocal;
+import org.apache.pig.pen.physicalOperators.POCounter;
+import org.apache.pig.pen.physicalOperators.POCogroup;
+import org.apache.pig.pen.physicalOperators.POCross;
+import org.apache.pig.pen.physicalOperators.POSplit;
+import org.apache.pig.pen.physicalOperators.POSplitOutput;
+import org.apache.pig.pen.physicalOperators.POStreamLocal;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.LOCogroup;
Copied:
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCogroup.java (from
r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCogroup.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCogroup.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCogroup.java
Wed Jan 6 17:27:17 2010
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators;
+package org.apache.pig.pen.physicalOperators;
import java.util.Comparator;
import java.util.Iterator;
Copied:
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCounter.java (from
r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/counters/POCounter.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCounter.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCounter.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/counters/POCounter.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/counters/POCounter.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCounter.java
Wed Jan 6 17:27:17 2010
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.pig.backend.local.executionengine.physicalLayer.counters;
+package org.apache.pig.pen.physicalOperators;
import java.util.List;
import java.util.Map;
Copied: hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCross.java
(from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCross.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCross.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCross.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCross.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCross.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POCross.java Wed
Jan 6 17:27:17 2010
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators;
+package org.apache.pig.pen.physicalOperators;
import java.util.Iterator;
import java.util.List;
Copied: hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplit.java
(from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplit.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplit.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplit.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplit.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplit.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplit.java Wed
Jan 6 17:27:17 2010
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators;
+package org.apache.pig.pen.physicalOperators;
import java.util.List;
Copied:
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplitOutput.java
(from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplitOutput.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplitOutput.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplitOutput.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplitOutput.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POSplitOutput.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POSplitOutput.java
Wed Jan 6 17:27:17 2010
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators;
+package org.apache.pig.pen.physicalOperators;
import java.util.Iterator;
import java.util.List;
Copied:
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POStreamLocal.java
(from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POStreamLocal.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POStreamLocal.java?p2=hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POStreamLocal.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POStreamLocal.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POStreamLocal.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/pen/physicalOperators/POStreamLocal.java
Wed Jan 6 17:27:17 2010
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators;
+package org.apache.pig.pen.physicalOperators;
import java.util.Properties;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java Wed Jan 6 17:27:17
2010
@@ -27,7 +27,7 @@
import junit.framework.TestCase;
import org.apache.pig.PigServer;
-import org.apache.pig.backend.local.datastorage.LocalSeekableInputStream;
+import org.apache.pig.test.utils.LocalSeekableInputStream;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLocalPOSplit.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLocalPOSplit.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestLocalPOSplit.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestLocalPOSplit.java Wed Jan 6
17:27:17 2010
@@ -37,7 +37,7 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.LocalLogToPhyTranslationVisitor;
+import org.apache.pig.pen.LocalLogToPhyTranslationVisitor;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPOCogroup.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPOCogroup.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPOCogroup.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPOCogroup.java Wed Jan 6
17:27:17 2010
@@ -32,7 +32,7 @@
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrangeForIllustrate;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORead;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POCogroup;
+import org.apache.pig.pen.physicalOperators.POCogroup;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPOCross.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPOCross.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPOCross.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPOCross.java Wed Jan 6
17:27:17 2010
@@ -28,7 +28,7 @@
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORead;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POCross;
+import org.apache.pig.pen.physicalOperators.POCross;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigLineRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigLineRecordReader.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigLineRecordReader.java
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigLineRecordReader.java Wed
Jan 6 17:27:17 2010
@@ -25,7 +25,7 @@
import java.io.PrintStream;
import java.lang.reflect.Field;
import org.apache.hadoop.io.Text;
-import org.apache.pig.backend.local.datastorage.LocalSeekableInputStream;
+import org.apache.pig.test.utils.LocalSeekableInputStream;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.BufferedPositionedInputStream;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=896552&r1=896551&r2=896552&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java Wed Jan 6
17:27:17 2010
@@ -21,8 +21,10 @@
import org.apache.pig.ExecType;
+import java.io.File;
import java.io.BufferedReader;
import java.io.FileReader;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -41,13 +43,13 @@
import org.apache.pig.PigServer;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.FileSpec;
-import org.apache.pig.backend.local.executionengine.LocalPigLauncher;
-import org.apache.pig.backend.local.executionengine.LocalPOStoreImpl;
-import
org.apache.pig.backend.local.executionengine.physicalLayer.counters.POCounter;
+import org.apache.pig.pen.physicalOperators.POCounter;
+import org.apache.pig.backend.executionengine.ExecException;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
+import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.test.utils.GenPhyOp;
import org.apache.pig.test.utils.GenRandomData;
import org.apache.pig.test.utils.TestHelper;
@@ -70,9 +72,17 @@
POProject proj;
PigServer pig;
POCounter pcount;
-
+
+ String inputFileName;
+ String outputFileName;
+
@Before
public void setUp() throws Exception {
+ pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pc = pig.getPigContext();
+ inputFileName = "/tmp/TestStore-" + new Random().nextLong() + ".txt";
+ outputFileName = "/tmp/TestStore-output-" + new Random().nextLong() +
".txt";
+ /*
st = GenPhyOp.topStoreOp();
pcount = new POCounter(new OperatorKey("", (new Random()).nextLong()));
fSpec = new FileSpec("file:/tmp/storeTest.txt",
@@ -83,17 +93,64 @@
pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
pc = pig.getPigContext();
- st.setStoreImpl(new LocalPOStoreImpl(pc));
+ //st.setStoreImpl(new LocalPOStoreImpl(pc));
proj = GenPhyOp.exprProject();
proj.setColumn(0);
proj.setResultType(DataType.TUPLE);
proj.setOverloaded(true);
List<PhysicalOperator> inps = new ArrayList<PhysicalOperator>();
+ */
}
@After
public void tearDown() throws Exception {
+ Util.deleteFile(cluster, inputFileName);
+ Util.deleteFile(cluster, outputFileName);
+ new File(outputFileName).delete();
+ }
+
+ private void storeAndCopyLocally(DataBag inpDB) throws Exception {
+ setUpInputFileOnCluster(inpDB);
+ String script = "a = load '" + inputFileName + "'; " +
+ "store a into '" + outputFileName + "' using PigStorage(':');"
+
+ "fs -ls /tmp";
+ pig.setBatchOn();
+ Util.registerMultiLineQuery(pig, script);
+ pig.executeBatch();
+ Util.copyFromClusterToLocal(cluster, outputFileName + "/part-00000",
outputFileName);
+ }
+
+ private void setUpInputFileOnCluster(DataBag inpD) throws IOException {
+ String[] data = new String[(int) inpD.size()];
+ int i = 0;
+ for (Tuple tuple : inpD) {
+ data[i] = toDelimitedString(tuple, "\t");
+ i++;
+ }
+ Util.createInputFile(cluster, inputFileName, data);
+ }
+
+ @SuppressWarnings("unchecked")
+ private String toDelimitedString(Tuple t, String delim) throws
ExecException {
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < t.size(); i++) {
+ Object field = t.get(i);
+ if(field == null) {
+ buf.append("");
+ } else {
+ if(field instanceof Map) {
+ Map<String, Object> m = (Map<String, Object>)field;
+ buf.append(DataType.mapToString(m));
+ } else {
+ buf.append(field.toString());
+ }
+ }
+
+ if (i != t.size() - 1)
+ buf.append(delim);
+ }
+ return buf.toString();
}
private PigStats store() throws Exception {
@@ -105,22 +162,19 @@
pp.connect(proj, pcount);
pp.connect(pcount, st);
pc.setExecType(ExecType.LOCAL);
- return new LocalPigLauncher().launchPig(pp, "TestStore", pc);
+ return new MapReduceLauncher().launchPig(pp, "TestStore", pc);
}
@Test
public void testStore() throws Exception {
inpDB = GenRandomData.genRandSmallTupDataBag(new Random(), 10, 100);
- Tuple t = new DefaultTuple();
- t.append(inpDB);
- proj.attachInput(t);
- assertTrue(store() != null);
-
+ storeAndCopyLocally(inpDB);
+
int size = 0;
- BufferedReader br = new BufferedReader(new
FileReader("/tmp/storeTest.txt"));
+ BufferedReader br = new BufferedReader(new FileReader(outputFileName));
for(String line=br.readLine();line!=null;line=br.readLine()){
String[] flds = line.split(":",-1);
- t = new DefaultTuple();
+ Tuple t = new DefaultTuple();
t.append(flds[0].compareTo("")!=0 ? flds[0] : null);
t.append(flds[1].compareTo("")!=0 ? Integer.parseInt(flds[1]) :
null);
@@ -132,23 +186,19 @@
++size;
}
assertEquals(true, size==inpDB.size());
- FileLocalizer.delete(fSpec.getFileName(), pc);
}
@Test
public void testStoreComplexData() throws Exception {
inpDB = GenRandomData.genRandFullTupTextDataBag(new Random(), 10, 100);
- Tuple t = new DefaultTuple();
- t.append(inpDB);
- proj.attachInput(t);
- assertTrue(store() != null);
+ storeAndCopyLocally(inpDB);
PigStorage ps = new PigStorage(":");
-
+
int size = 0;
- BufferedReader br = new BufferedReader(new
FileReader("/tmp/storeTest.txt"));
+ BufferedReader br = new BufferedReader(new FileReader(outputFileName));
for(String line=br.readLine();line!=null;line=br.readLine()){
String[] flds = line.split(":",-1);
- t = new DefaultTuple();
+ Tuple t = new DefaultTuple();
t.append(flds[0].compareTo("")!=0 ?
ps.bytesToBag(flds[0].getBytes()) : null);
t.append(flds[1].compareTo("")!=0 ?
ps.bytesToCharArray(flds[1].getBytes()) : null);
t.append(flds[2].compareTo("")!=0 ?
ps.bytesToCharArray(flds[2].getBytes()) : null);
@@ -163,7 +213,6 @@
++size;
}
assertEquals(true, size==inpDB.size());
- FileLocalizer.delete(fSpec.getFileName(), pc);
}
@Test
@@ -171,19 +220,17 @@
Tuple inputTuple = GenRandomData.genRandSmallBagTextTupleWithNulls(new
Random(), 10, 100);
inpDB = DefaultBagFactory.getInstance().newDefaultBag();
inpDB.add(inputTuple);
- Tuple t = new DefaultTuple();
- t.append(inpDB);
- proj.attachInput(t);
- assertTrue(store() != null);
+ storeAndCopyLocally(inpDB);
+
PigStorage ps = new PigStorage(":");
int size = 0;
- BufferedReader br = new BufferedReader(new
FileReader("/tmp/storeTest.txt"));
+ BufferedReader br = new BufferedReader(new FileReader(outputFileName));
for(String line=br.readLine();line!=null;line=br.readLine()){
System.err.println("Complex data: ");
System.err.println(line);
String[] flds = line.split(":",-1);
- t = new DefaultTuple();
+ Tuple t = new DefaultTuple();
t.append(flds[0].compareTo("")!=0 ?
ps.bytesToBag(flds[0].getBytes()) : null);
t.append(flds[1].compareTo("")!=0 ?
ps.bytesToCharArray(flds[1].getBytes()) : null);
t.append(flds[2].compareTo("")!=0 ?
ps.bytesToCharArray(flds[2].getBytes()) : null);
@@ -198,7 +245,6 @@
assertTrue(TestHelper.tupleEquals(inputTuple, t));
++size;
}
- FileLocalizer.delete(fSpec.getFileName(), pc);
}
@Test
Copied:
hadoop/pig/trunk/test/org/apache/pig/test/utils/LocalSeekableInputStream.java
(from r896256,
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalSeekableInputStream.java)
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/LocalSeekableInputStream.java?p2=hadoop/pig/trunk/test/org/apache/pig/test/utils/LocalSeekableInputStream.java&p1=hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalSeekableInputStream.java&r1=896256&r2=896552&rev=896552&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalSeekableInputStream.java
(original)
+++
hadoop/pig/trunk/test/org/apache/pig/test/utils/LocalSeekableInputStream.java
Wed Jan 6 17:27:17 2010
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.pig.backend.local.datastorage;
+package org.apache.pig.test.utils;
import java.io.RandomAccessFile;
import java.io.File;
