Author: daijy
Date: Mon Sep 21 16:55:35 2009
New Revision: 817319
URL: http://svn.apache.org/viewvc?rev=817319&view=rev
Log:
PIG-891: Fixing dfs statement for Pig
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=817319&r1=817318&r2=817319&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Sep 21 16:55:35 2009
@@ -26,6 +26,8 @@
IMPROVEMENTS
+PIG-891: Fixing dfs statement for Pig (zjffdu via daijy).
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=817319&r1=817318&r2=817319&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Mon Sep 21
16:55:35 2009
@@ -47,6 +47,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.JobID;
@@ -57,6 +58,7 @@
import org.apache.pig.backend.datastorage.DataStorageException;
import org.apache.pig.backend.datastorage.ElementDescriptor;
import org.apache.pig.backend.executionengine.ExecutionEngine;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
@@ -191,6 +193,7 @@
mDfs = mPigServer.getPigContext().getDfs();
mLfs = mPigServer.getPigContext().getLfs();
mConf = mPigServer.getPigContext().getProperties();
+ shell = new FsShell(ConfigurationUtil.toConfiguration(mConf));
// TODO: this violates the abstraction layer decoupling between
// front end and back end and needs to be changed.
@@ -731,6 +734,14 @@
}
}
+ protected void processFsCommand(String[] cmdTokens) throws IOException{
+ try {
+ shell.run(cmdTokens);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
private class ExplainState {
public long mTime;
public int mCount;
@@ -764,4 +775,5 @@
private ExplainState mExplain;
private int mNumFailedJobs;
private int mNumSucceededJobs;
+ private FsShell shell;
}
Modified:
hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=817319&r1=817318&r2=817319&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
Mon Sep 21 16:55:35 2009
@@ -65,6 +65,8 @@
abstract protected void printAliases() throws IOException;
+ abstract protected void processFsCommand(String[] cmdTokens) throws
IOException;
+
abstract protected void processDescribe(String alias) throws
IOException;
abstract protected void processExplain(String alias, String script,
boolean isVerbose, String format, String target, List<String> params,
List<String> files) throws IOException, ParseException;
@@ -130,6 +132,7 @@
// commands
TOKEN: {<CAT: "cat">}
+TOKEN: {<FS: "fs">}
TOKEN: {<CD: "cd">}
TOKEN: {<COPY: "cp">}
TOKEN: {<COPYFROMLOCAL: "copyFromLocal">}
@@ -360,6 +363,7 @@
{
Token t1, t2;
String val = null;
+ List<String> cmdTokens = new ArrayList<String>();
}
{
@@ -367,6 +371,23 @@
<EOL>
{prompt();}
|
+ <FS>
+ (
+ t1 = GetPath()
+ {
+ cmdTokens.add(t1.image);
+ while(true){
+ try{
+ t1=GetPath();
+ cmdTokens.add(t1.image);
+ }catch(ParseException e){
+ break;
+ }
+ }
+ processFsCommand(cmdTokens.toArray(new
String[cmdTokens.size()]));
+ }
+ )+
+ |
<CAT>
(
t1 = GetPath()
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=817319&r1=817318&r2=817319&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Mon Sep 21
16:55:35 2009
@@ -26,6 +26,7 @@
import org.apache.pig.ExecType;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.impl.PigContext;
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.pigscript.parser.ParseException;
@@ -709,4 +710,39 @@
assertFalse(server.existsFile("done"));
assertTrue(caught);
}
+
+ public void testFsCommand(){
+
+ try {
+ PigServer server = new
PigServer(ExecType.MAPREDUCE,cluster.getProperties());
+ PigContext context = server.getPigContext();
+
+ String strCmd =
+ "fs -ls /;"
+ +"fs -mkdir /tmp;"
+ +"fs -mkdir /tmp/foo;"
+ +"cd /tmp;"
+ +"fs -rmr bar; fs -rmr foo/baz;"
+ +"fs -copyFromLocal test/org/apache/pig/test/data/passwd bar;"
+ +"a = load 'bar';"
+ +"cd foo;"
+ +"store a into 'baz';"
+ +"cd /;"
+ +"fs -ls .;"
+ +"fs -rm /tmp/foo/baz;";
+
+ ByteArrayInputStream cmd = new
ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+
+ } catch (ExecException e) {
+ e.printStackTrace();
+ fail();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
}
