Author: thejas
Date: Thu Sep 30 16:43:21 2010
New Revision: 1003148

URL: http://svn.apache.org/viewvc?rev=1003148&view=rev
Log:
PIG-1650: pig grunt shell breaks for many commands like perl , awk ,
pipe , 'ls -l' etc  (nrai via thejas)

Added:
    hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/util/StringUtils.java
Modified:
    hadoop/pig/branches/branch-0.8/CHANGES.txt
    
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/grunt/GruntParser.java
    
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
    hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java

Modified: hadoop/pig/branches/branch-0.8/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/CHANGES.txt?rev=1003148&r1=1003147&r2=1003148&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.8/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.8/CHANGES.txt Thu Sep 30 16:43:21 2010
@@ -198,6 +198,9 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1650: pig grunt shell breaks for many commands like perl , awk ,
+pipe , 'ls -l' etc  (nrai via thejas)
+
 PIG-1649: FRJoin fails to compute number of input files for replicated input 
(thejas)
 
 PIG-1637: Combiner not use because optimizor inserts a foreach between group

Modified: 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=1003148&r1=1003147&r2=1003148&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 (original)
+++ 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 Thu Sep 30 16:43:21 2010
@@ -56,6 +56,7 @@ import org.apache.pig.impl.streaming.Str
 import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec;
 import 
org.apache.pig.impl.logicalLayer.validators.LogicalPlanValidationExecutor;
 import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.util.StringUtils;
 import org.apache.pig.StreamToPig;
 import org.apache.pig.PigToStream;
 import org.apache.pig.builtin.PigStreaming;
@@ -796,139 +797,6 @@ public class QueryParser {
 }
 
 
-class StringUtils {
-    
-       public static String unescapeInputString(String input)  {
-
-            if (input == null) {
-                return new String() ;
-            }
-            
-            // Needed variables
-            // preset the size so our StringBuilders don't have to grow
-            int inputlength = input.length();       
-            StringBuilder unicode = new StringBuilder(4);
-            StringBuilder output = new StringBuilder(inputlength) ;
-            boolean hadSlash = false;
-            boolean inUnicode = false;
-            
-            // The main loop
-            for (int i = 0; i < inputlength; i++) {
-                char ch = input.charAt(i);
-                // currently doing unicode mode
-                if (inUnicode) {
-                    unicode.append(ch);
-                    if (unicode.length() == 4) {
-                        // unicode now contains the four hex digits
-                        try {
-                            int value = Integer.parseInt(unicode.toString(), 
0x10);
-                            output.append((char) value) ;
-                            // reuse the StringBuilder
-                            unicode.setLength(0);
-                            inUnicode = false;
-                            hadSlash = false;
-                        } catch (NumberFormatException nfe) {
-                            throw new RuntimeException("Unable to parse 
unicode value: " + unicode, nfe);
-                        }
-                    }
-                    continue;
-                }
-                if (hadSlash) {
-                    // handle an escaped value
-                    hadSlash = false;
-                    switch (ch) {
-                        case '\\':
-                            output.append('\\');
-                            break;
-                        case '\'':
-                            output.append('\'');
-                            break;
-                        case 'r':
-                            output.append('\r');
-                            break;
-                        case 'f':
-                            output.append('\f');
-                            break;
-                        case 't':
-                            output.append('\t');
-                            break;
-                        case 'n':
-                            output.append('\n');
-                            break;
-                        case 'b':
-                            output.append('\b');
-                            break;
-                        case 'u':
-                            {
-                                // switch to unicode mode
-                                inUnicode = true;
-                                break;
-                            }
-                        default :
-                            output.append(ch);
-                            break;
-                    }
-                    continue;
-                } else if (ch == '\\') {
-                    hadSlash = true;
-                    continue;
-                }
-                output.append(ch);
-            }
-            
-            return output.toString() ;
-        }
-        
-        public static String join(AbstractCollection<String> s, String 
delimiter) {
-            if (s.isEmpty()) return "";
-            Iterator<String> iter = s.iterator();
-            StringBuffer buffer = new StringBuffer(iter.next());
-            while (iter.hasNext()) {
-                buffer.append(delimiter);
-                buffer.append(iter.next());
-            }
-            return buffer.toString();
-        }
-        
-            
-        public static String[] getPathStrings(String commaSeparatedPaths) {
-            int length = commaSeparatedPaths.length();
-            int curlyOpen = 0;
-            int pathStart = 0;
-            boolean globPattern = false;
-            List<String> pathStrings = new ArrayList<String>();
-        
-            for (int i=0; i<length; i++) {
-                char ch = commaSeparatedPaths.charAt(i);
-                switch(ch) {
-                    case '{' : {
-                        curlyOpen++;
-                        if (!globPattern) {
-                            globPattern = true;
-                        }
-                        break;
-                    }
-                    case '}' : {
-                        curlyOpen--;
-                        if (curlyOpen == 0 && globPattern) {
-                            globPattern = false;
-                        }
-                        break;
-                    }
-                    case ',' : {
-                        if (!globPattern) {
-                            
pathStrings.add(commaSeparatedPaths.substring(pathStart, i));
-                            pathStart = i + 1 ;
-                        }
-                        break;
-                    }
-                }
-            }
-            pathStrings.add(commaSeparatedPaths.substring(pathStart, length));
-        
-            return pathStrings.toArray(new String[0]);
-        }
-}
 
 class FunctionType {
     public static final byte UNKNOWNFUNC = 0;

Added: 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/util/StringUtils.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/util/StringUtils.java?rev=1003148&view=auto
==============================================================================
--- 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/util/StringUtils.java 
(added)
+++ 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/impl/util/StringUtils.java 
Thu Sep 30 16:43:21 2010
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.impl.util;
+
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class StringUtils {
+    
+    public static String unescapeInputString(String input)  {
+
+         if(input == null){
+            return "";
+         }
+         // Needed variables
+         // preset the size so our StringBuilders don't have to grow
+         int inputlength = input.length();       
+         StringBuilder unicode = new StringBuilder(4);
+         StringBuilder output = new StringBuilder(inputlength) ;
+         boolean hadSlash = false;
+         boolean inUnicode = false;
+         
+         // The main loop
+         for (int i = 0; i < inputlength; i++) {
+             char ch = input.charAt(i);
+             // currently doing unicode mode
+             if (inUnicode) {
+                 unicode.append(ch);
+                 if (unicode.length() == 4) {
+                     // unicode now contains the four hex digits
+                     try {
+                         int value = Integer.parseInt(unicode.toString(), 
0x10);
+                         output.append((char) value) ;
+                         // reuse the StringBuilder
+                         unicode.setLength(0);
+                         inUnicode = false;
+                         hadSlash = false;
+                     } catch (NumberFormatException nfe) {
+                         throw new RuntimeException("Unable to parse unicode 
value: " + unicode, nfe);
+                     }
+                 }
+                 continue;
+             }
+             if (hadSlash) {
+                 // handle an escaped value
+                 hadSlash = false;
+                 switch (ch) {
+                     case '\\':
+                         output.append('\\');
+                         break;
+                     case '\'':
+                         output.append('\'');
+                         break;
+                     case 'r':
+                         output.append('\r');
+                         break;
+                     case 'f':
+                         output.append('\f');
+                         break;
+                     case 't':
+                         output.append('\t');
+                         break;
+                     case 'n':
+                         output.append('\n');
+                         break;
+                     case 'b':
+                         output.append('\b');
+                         break;
+                     case 'u':
+                         {
+                             // switch to unicode mode
+                             inUnicode = true;
+                             break;
+                         }
+                     default :
+                         output.append(ch);
+                         break;
+                 }
+                 continue;
+             } else if (ch == '\\') {
+                 hadSlash = true;
+                 continue;
+             }
+             output.append(ch);
+         }
+         
+         return output.toString() ;
+     }
+     
+     public static String join(AbstractCollection<String> s, String delimiter) 
{
+         if (s.isEmpty()) return "";
+         Iterator<String> iter = s.iterator();
+         StringBuffer buffer = new StringBuffer(iter.next());
+         while (iter.hasNext()) {
+             buffer.append(delimiter);
+             buffer.append(iter.next());
+         }
+         return buffer.toString();
+     }
+     
+         
+     public static String[] getPathStrings(String commaSeparatedPaths) {
+         int length = commaSeparatedPaths.length();
+         int curlyOpen = 0;
+         int pathStart = 0;
+         boolean globPattern = false;
+         List<String> pathStrings = new ArrayList<String>();
+     
+         for (int i=0; i<length; i++) {
+             char ch = commaSeparatedPaths.charAt(i);
+             switch(ch) {
+                 case '{' : {
+                     curlyOpen++;
+                     if (!globPattern) {
+                         globPattern = true;
+                     }
+                     break;
+                 }
+                 case '}' : {
+                     curlyOpen--;
+                     if (curlyOpen == 0 && globPattern) {
+                         globPattern = false;
+                     }
+                     break;
+                 }
+                 case ',' : {
+                     if (!globPattern) {
+                         
pathStrings.add(commaSeparatedPaths.substring(pathStart, i));
+                         pathStart = i + 1 ;
+                     }
+                     break;
+                 }
+             }
+         }
+         pathStrings.add(commaSeparatedPaths.substring(pathStart, length));
+     
+         return pathStrings.toArray(new String[0]);
+     }
+}

Modified: 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/grunt/GruntParser.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1003148&r1=1003147&r2=1003148&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/grunt/GruntParser.java 
(original)
+++ 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/grunt/GruntParser.java 
Thu Sep 30 16:43:21 2010
@@ -909,12 +909,8 @@ public class GruntParser extends PigScri
     
     @Override
     protected void processShCommand(String[] cmdTokens) throws IOException{
-        StringBuilder builder = new StringBuilder();
-        for (String token:cmdTokens){
-            builder.append(token + " ");
-        }
         try {
-            Process executor = Runtime.getRuntime().exec(builder.toString());
+            Process executor = Runtime.getRuntime().exec(cmdTokens);
             StreamPrinter outPrinter = new 
StreamPrinter(executor.getInputStream(), null, System.out);
             StreamPrinter errPrinter = new 
StreamPrinter(executor.getErrorStream(), null, System.err);
 

Modified: 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1003148&r1=1003147&r2=1003148&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
 (original)
+++ 
hadoop/pig/branches/branch-0.8/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
 Thu Sep 30 16:43:21 2010
@@ -34,6 +34,7 @@ import java.io.InputStream;
 import java.util.Stack;
 import java.util.List;
 import java.util.ArrayList;
+import org.apache.pig.impl.util.StringUtils;
 
 import jline.ConsoleReader;
 
@@ -405,13 +406,13 @@ void parse() throws IOException:
        |
        <SH>
        (
-       t1 = GetPath()
+       t1 = GetValue()
        {
                cmdTokens.add(t1.image);
                while(true){
                        try{
-                               t1=GetPath();
-                               cmdTokens.add(t1.image);
+                               t1=GetValue();
+                               
cmdTokens.add(StringUtils.unescapeInputString(unquote(t1.image)));
                        }catch(ParseException e){
                                break;
                        }

Modified: hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java?rev=1003148&r1=1003147&r2=1003148&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java 
(original)
+++ hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java Thu 
Sep 30 16:43:21 2010
@@ -869,6 +869,45 @@ public class TestGrunt extends TestCase 
             grunt = new Grunt(new BufferedReader(reader), context);
             grunt.exec();
             assertFalse(new File("test_shell_tmp").exists());
+            
+            strCmd = "sh bash -c 'echo hello world > 
tempShFileToTestShCommand'";
+            cmd = new ByteArrayInputStream(strCmd.getBytes());
+            reader = new InputStreamReader(cmd);
+            grunt = new Grunt(new BufferedReader(reader), context);
+            grunt.exec();
+            BufferedReader fileReader = null;
+            fileReader = new BufferedReader(new 
FileReader("tempShFileToTestShCommand"));
+            assertTrue(fileReader.readLine().equalsIgnoreCase("hello world"));
+            fileReader.close();
+            strCmd = "sh rm tempShFileToTestShCommand";
+            cmd = new ByteArrayInputStream(strCmd.getBytes());
+            reader = new InputStreamReader(cmd);
+            grunt = new Grunt(new BufferedReader(reader), context);
+            grunt.exec();
+            assertFalse(new File("tempShFileToTestShCommand").exists());
+
+            strCmd = "sh bash -c 'touch TouchedFileInsideGrunt_61 | ls | grep 
TouchedFileInsideGrunt_61 > fileContainingTouchedFileInsideGruntShell_71'";
+            cmd = new ByteArrayInputStream(strCmd.getBytes());
+            reader = new InputStreamReader(cmd);
+            grunt = new Grunt(new BufferedReader(reader), context);
+            grunt.exec();
+            fileReader = new BufferedReader(new 
FileReader("fileContainingTouchedFileInsideGruntShell_71"));
+            
assertTrue(fileReader.readLine().equals("TouchedFileInsideGrunt_61"));
+            fileReader.close();
+            strCmd = "sh bash -c 'rm 
fileContainingTouchedFileInsideGruntShell_71'";
+            cmd = new ByteArrayInputStream(strCmd.getBytes());
+            reader = new InputStreamReader(cmd);
+            grunt = new Grunt(new BufferedReader(reader), context);
+            grunt.exec();
+            assertFalse(new 
File("fileContainingTouchedFileInsideGruntShell_71").exists());
+            strCmd = "sh bash -c 'rm TouchedFileInsideGrunt_61'";
+            cmd = new ByteArrayInputStream(strCmd.getBytes());
+            reader = new InputStreamReader(cmd);
+            grunt = new Grunt(new BufferedReader(reader), context);
+            grunt.exec();
+            assertFalse(new File("TouchedFileInsideGrunt_61").exists());
+         
+            
         } catch (ExecException e) {
             e.printStackTrace();
             fail();
@@ -877,7 +916,7 @@ public class TestGrunt extends TestCase 
             fail();
         }
     }
-    
+
     @Test
     public void testSetPriority() throws Throwable {
         PigServer server = new PigServer(ExecType.MAPREDUCE, 
cluster.getProperties());


Reply via email to