Author: daijy Date: Mon Sep 21 16:55:35 2009 New Revision: 817319 URL: http://svn.apache.org/viewvc?rev=817319&view=rev Log: PIG-891: Fixing dfs statement for Pig
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=817319&r1=817318&r2=817319&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Mon Sep 21 16:55:35 2009 @@ -26,6 +26,8 @@ IMPROVEMENTS +PIG-891: Fixing dfs statement for Pig (zjffdu via daijy). + OPTIMIZATIONS BUG FIXES Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=817319&r1=817318&r2=817319&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Mon Sep 21 16:55:35 2009 @@ -47,6 +47,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.JobID; @@ -57,6 +58,7 @@ import org.apache.pig.backend.datastorage.DataStorageException; import org.apache.pig.backend.datastorage.ElementDescriptor; import org.apache.pig.backend.executionengine.ExecutionEngine; +import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; import org.apache.pig.backend.executionengine.ExecJob; import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS; @@ -191,6 +193,7 @@ mDfs = mPigServer.getPigContext().getDfs(); mLfs = mPigServer.getPigContext().getLfs(); mConf = mPigServer.getPigContext().getProperties(); + shell = new FsShell(ConfigurationUtil.toConfiguration(mConf)); // TODO: this violates the abstraction layer decoupling between // front end and back end and needs to be changed. @@ -731,6 +734,14 @@ } } + protected void processFsCommand(String[] cmdTokens) throws IOException{ + try { + shell.run(cmdTokens); + } catch (Exception e) { + throw new IOException(e); + } + } + private class ExplainState { public long mTime; public int mCount; @@ -764,4 +775,5 @@ private ExplainState mExplain; private int mNumFailedJobs; private int mNumSucceededJobs; + private FsShell shell; } Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=817319&r1=817318&r2=817319&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Mon Sep 21 16:55:35 2009 @@ -65,6 +65,8 @@ abstract protected void printAliases() throws IOException; + abstract protected void processFsCommand(String[] cmdTokens) throws IOException; + abstract protected void processDescribe(String alias) throws IOException; abstract protected void processExplain(String alias, String script, boolean isVerbose, String format, String target, List<String> params, List<String> files) throws IOException, ParseException; @@ -130,6 +132,7 @@ // commands TOKEN: {<CAT: "cat">} +TOKEN: {<FS: "fs">} TOKEN: {<CD: "cd">} TOKEN: {<COPY: "cp">} TOKEN: {<COPYFROMLOCAL: "copyFromLocal">} @@ -360,6 +363,7 @@ { Token t1, t2; String val = null; + List<String> cmdTokens = new ArrayList<String>(); } { @@ -367,6 +371,23 @@ <EOL> {prompt();} | + <FS> + ( + t1 = GetPath() + { + cmdTokens.add(t1.image); + while(true){ + try{ + t1=GetPath(); + cmdTokens.add(t1.image); + }catch(ParseException e){ + break; + } + } + processFsCommand(cmdTokens.toArray(new String[cmdTokens.size()])); + } + )+ + | <CAT> ( t1 = GetPath() Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=817319&r1=817318&r2=817319&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Mon Sep 21 16:55:35 2009 @@ -26,6 +26,7 @@ import org.apache.pig.ExecType; import org.apache.pig.PigException; import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.impl.PigContext; import org.apache.pig.tools.grunt.Grunt; import org.apache.pig.tools.pigscript.parser.ParseException; @@ -709,4 +710,39 @@ assertFalse(server.existsFile("done")); assertTrue(caught); } + + public void testFsCommand(){ + + try { + PigServer server = new PigServer(ExecType.MAPREDUCE,cluster.getProperties()); + PigContext context = server.getPigContext(); + + String strCmd = + "fs -ls /;" + +"fs -mkdir /tmp;" + +"fs -mkdir /tmp/foo;" + +"cd /tmp;" + +"fs -rmr bar; fs -rmr foo/baz;" + +"fs -copyFromLocal test/org/apache/pig/test/data/passwd bar;" + +"a = load 'bar';" + +"cd foo;" + +"store a into 'baz';" + +"cd /;" + +"fs -ls .;" + +"fs -rm /tmp/foo/baz;"; + + ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes()); + InputStreamReader reader = new InputStreamReader(cmd); + + Grunt grunt = new Grunt(new BufferedReader(reader), context); + grunt.exec(); + + } catch (ExecException e) { + e.printStackTrace(); + fail(); + } catch (Throwable e) { + e.printStackTrace(); + fail(); + } + } }