olga
Thu, 24 Apr 2008 16:00:51 -0700
Author: olga Date: Thu Apr 24 16:00:10 2008 New Revision: 651453 URL: http://svn.apache.org/viewvc?rev=651453&view=rev Log: PIG-200: changes to the definition of parameter name Added: incubator/pig/trunk/test/org/apache/pig/test/data/inputNoVars.pig Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj incubator/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=651453&r1=651452&r2=651453&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Thu Apr 24 16:00:10 2008 @@ -252,3 +252,4 @@ PIG-218: Fixed param generation to work with arbitrary commands + PIG-220: Fixed definition of parameter name for param substitution Modified: incubator/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=651453&r1=651452&r2=651453&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj (original) +++ incubator/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj Thu Apr 24 16:00:10 2008 @@ -70,7 +70,8 @@ TOKEN : { - <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> | <SPECIALCHAR>)*> +// <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> | <SPECIALCHAR>)*> + <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*> | <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r"] (~["\n","\r"])* > | Modified: incubator/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=651453&r1=651452&r2=651453&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj (original) +++ incubator/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Thu Apr 24 16:00:10 2008 @@ -78,7 +78,7 @@ TOKEN : { - <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> | <SPECIALCHAR>)*> + <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*> | <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* "'") > | Modified: incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=651453&r1=651452&r2=651453&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original) +++ incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Apr 24 16:00:10 2008 @@ -199,7 +199,7 @@ return streamData.trim(); } - private static String id_regex = "\\$[a-zA-Z_][a-zA-Z_0-9]*"; + private static String id_regex = "\\$[_]*[a-zA-Z][a-zA-Z_0-9]*"; public String substitute(String line) { int index = line.indexOf('$'); Modified: incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=651453&r1=651452&r2=651453&view=diff ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original) +++ incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Thu Apr 24 16:00:10 2008 @@ -1190,7 +1190,7 @@ */ @Test public void testCmdlineParamWithInlineCmd() throws Exception{ - log.info("Starting test testCmdlineParam() ..."); + log.info("Starting test testCmdlineParamWithInlineCmd() ..."); try { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); @@ -1202,6 +1202,56 @@ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); + } catch (ParseException e) { + fail ("Got ParseException : " + e.getMessage()); + } catch (RuntimeException e) { + fail ("Got RuntimeException : " + e.getMessage()); + } catch (Error e) { + fail ("Got error : " + e.getMessage()); + } + + log.info("Done"); + + } + + /* Test case 31 + * Use of inline command + */ + @Test + public void testNoVars() throws Exception{ + log.info("Starting test testNoVars() ..."); + try { + ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); + pigIStream = new BufferedReader(new FileReader(basedir + "/inputNoVars.pig")); + pigOStream = new FileWriter(basedir + "/output1.pig"); + + String[] arg = null; + String[] argFiles = null; + ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); + + FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); + pigExResultStream = new FileInputStream(basedir + "/inputNoVars.pig"); BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); Added: incubator/pig/trunk/test/org/apache/pig/test/data/inputNoVars.pig URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/data/inputNoVars.pig?rev=651453&view=auto ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/data/inputNoVars.pig (added) +++ incubator/pig/trunk/test/org/apache/pig/test/data/inputNoVars.pig Thu Apr 24 16:00:10 2008 @@ -0,0 +1,6 @@ + +register /home/y/lib/java/pigtest/testudf.jar; +A = load '/user/pig/tests/data/singlefile/textdoc' using TextLoader(); +define X `perl -ne 'chomp $_; print "$_\n"'` output (stdout using org.apache.pig.test.udf.storefunc.StringStore()); +B = stream A through X; +store B into '/user/pig/tests/results/olgan.1209067990/DefineClause_4.out'; Modified: incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=651453&r1=651452&r2=651453&view=diff ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig (original) +++ incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig Thu Apr 24 16:00:10 2008 @@ -1,11 +1,6 @@ -aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01'); -bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400; -a = foreach bb generate $0,$12,$7; - ---generate inactive accts -inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct'; -grpInactiveAcct = group inactiveAccounts all; -countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); } -store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount'; +register /home/y/lib/java/pigtest/testudf.jar; +A = load '/user/pig/tests/data/singlefile/textdoc' using TextLoader(); +define X `perl -ne 'chomp $_; print "$_\n"'` output (stdout using org.apache.pig.test.udf.storefunc.StringStore()); +B = stream A through X; +store B into '/user/pig/tests/results/olgan.1209067990/DefineClause_4.out';