olga
Thu, 24 Apr 2008 09:29:00 -0700
Author: olga Date: Thu Apr 24 09:28:12 2008 New Revision: 651307 URL: http://svn.apache.org/viewvc?rev=651307&view=rev Log: PIG-218: fixed parameter generation code to work with arbitrary commands Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=651307&r1=651306&r2=651307&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Thu Apr 24 09:28:12 2008 @@ -244,9 +244,11 @@ PIG-213: Remove non-static references to logger from data bags and tuples, as it causes significant overhead (vgeschel via gates). - PIG-216 Fix streaming to work with commands that use unix pipes (acmurthy + PIG-216: Fix streaming to work with commands that use unix pipes (acmurthy via gates). - PIG-207 Fix illustrate command to work in mapreduce mode (shubhamc via + PIG-207: Fix illustrate command to work in mapreduce mode (shubhamc via gates). + + PIG-218: Fixed param generation to work with arbitrary commands Modified: incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=651307&r1=651306&r2=651307&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original) +++ incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Apr 24 09:28:12 2008 @@ -142,7 +142,16 @@ String streamError=""; try { log.info("Executing command : " + cmd); - p = Runtime.getRuntime().exec(cmd,null); + // we can't use exec directly since it does not handle + // case like foo -c "bar bar" correctly. It splits on white spaces even in presents of quotes + String[] cmdArgs = new String[3]; + cmdArgs[0] = "bash"; + cmdArgs[1] = "-c"; + StringBuffer sb = new StringBuffer("exec "); + sb.append(cmd); + cmdArgs[2] = sb.toString(); + + p = Runtime.getRuntime().exec(cmdArgs); } catch (IOException e) { RuntimeException rte = new RuntimeException("IO Exception while executing shell command : "+e.getMessage() , e); Modified: incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=651307&r1=651306&r2=651307&view=diff ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original) +++ incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Thu Apr 24 09:28:12 2008 @@ -1185,4 +1185,53 @@ } + /* Test case 31 + * Use of inline command + */ + @Test + public void testCmdlineParamWithInlineCmd() throws Exception{ + log.info("Starting test testCmdlineParam() ..."); + try { + ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); + pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); + pigOStream = new FileWriter(basedir + "/output1.pig"); + + String[] arg = {"date=`/usr/local/bin/perl -e 'print \"20080228\n20070101\"' | head -n 1`"}; + String[] argFiles = null; + ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); + + FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); + pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); + } catch (ParseException e) { + fail ("Got ParseException : " + e.getMessage()); + } catch (RuntimeException e) { + fail ("Got RuntimeException : " + e.getMessage()); + } catch (Error e) { + fail ("Got error : " + e.getMessage()); + } + + log.info("Done"); + + } } Modified: incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=651307&r1=651306&r2=651307&view=diff ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig (original) +++ incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig Thu Apr 24 09:28:12 2008 @@ -4,7 +4,7 @@ a = foreach bb generate $0,$12,$7; --generate inactive accts -inactiveAccounts = filter a by ($1 neq '$column1') and ($1 == '2') parallel 400; +inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct'; grpInactiveAcct = group inactiveAccounts all; countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }