Author: olga
Date: Wed Jun 3 22:00:16 2009
New Revision: 781599
URL: http://svn.apache.org/viewvc?rev=781599&view=rev
Log:
PIG-564: problem with parameter substitution and special charachters (olgan)
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt
hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Jun 3 22:00:16 2009
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+
+
Pig Change Log
Trunk (unreleased changes)
@@ -26,6 +28,8 @@
PIG-825: PIG_HADOOP_VERSION should be set to 18 (dvryaboy via gates).
+PIG-795: support for SAMPLE command (ericg via olgan)
+
PIG-619: Create one InputSplit even when the input file is zero length
so that hadoop runs maps and creates output for the next
job (gates).
@@ -37,19 +41,27 @@
PIG-712: Added utility functions to create schemas for tuples and bags (zjffdu
via gates).
+PIG-652: Adapt changes in store interface to multi-query changes (hagleitn via
gates).
+
PIG-775: PORelationToExprProject should create a NonSpillableDataBag to create
empty bags (pradeepkth)
PIG-741: Allow limit to be nested in a foreach.
+PIG-627: multiquery support phase 3 (hagleitn and Richard Ding via olgan)
+
PIG-743: To implement clover (gkesavan)
PIG-701: Implement IVY for resolving pig dependencies (gkesavan)
PIG-626: Add access to hadoop counters (shubhamc via gates).
+PIG-627: multiquery support phase 1 and phase 2 (hagleitn and Richard Ding via
pradeepkth)
+
BUG FIXES
+PIG-564: problem with parameter substitution and special charachters (olgan)
+
PIG-802: PERFORMANCE: not creating bags for ORDER BY (serakesh via olgan)
PIG-816: PigStorage() does not accept Unicode characters in its contructor
(pradeepkth)
@@ -650,10 +662,3 @@
PIG-284: target for building source jar (oae via olgan)
- PIG-627: multiquery support phase 1 and phase 2 (hagleitn and Richard Ding
via pradeepkth)
-
- PIG-652: Adapt changes in store interface to multi-query changes (hagleitn
via gates).
-
- PIG-627: multiquery support phase 3 (hagleitn and Richard Ding via olgan)
-
- PIG-795: support for SAMPLE command (ericg via olgan)
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj Wed Jun
3 22:00:16 2009
@@ -74,7 +74,7 @@
// <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> |
<SPECIALCHAR>)*>
<IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
|
- <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " "
, "\n" , "\t" , "\r"] (~["\n","\r"])* >
+ <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " ,
"\n" , "\t" , "\r"] (~["\n","\r"])* >
|
<LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)*
"'") >
|
@@ -100,11 +100,17 @@
id=<IDENTIFIER>
<EQUALS>
(
- val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image);}
+ val=<OTHER> { pc.processOrdLine(id.image , val.image);}
|
- val=<ORD_LINE> {pc.processOrdLine(id.image , val.image);}
+ val=<IDENTIFIER>
+ {s = val.image;}
+ (
+ val =<OTHER>
+ {s += val.image;}
+ )?
+ {pc.processOrdLine(id.image , s);}
|
- val=<SHELLCMD> { pc.processShellCmd(id.image , val.image); }
+ val=<SHELLCMD> { pc.processShellCmd(id.image , val.image);}
|
val=<LITERAL> { s = unquote(val.image);
pc.processOrdLine(id.image , s); }
)
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Wed
Jun 3 22:00:16 2009
@@ -85,7 +85,7 @@
|
<SHELLCMD: "`" (~["`"])* "`" >
|
- <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " "
, "\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
+ <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " ,
"\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
}
@@ -102,9 +102,15 @@
(
id=<IDENTIFIER>
(
- val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image,
overwrite);}
+ val=<OTHER> { pc.processOrdLine(id.image , val.image);}
|
- val=<ORD_LINE> {pc.processOrdLine(id.image , val.image,
overwrite);}
+ val=<IDENTIFIER>
+ {s = val.image;}
+ (
+ val =<OTHER>
+ {s += val.image;}
+ )?
+ {pc.processOrdLine(id.image , s, overwrite);}
|
val=<SHELLCMD> { pc.processShellCmd(id.image , val.image,
overwrite); }
|
@@ -117,9 +123,15 @@
(
id=<IDENTIFIER>
(
- val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image,
overwrite);}
+ val=<OTHER> { pc.processOrdLine(id.image , val.image);}
|
- val=<ORD_LINE> {pc.processOrdLine(id.image , val.image,
overwrite);}
+ val=<IDENTIFIER>
+ {s = val.image;}
+ (
+ val =<OTHER>
+ {s += val.image;}
+ )?
+ {pc.processOrdLine(id.image , s, overwrite);}
|
val=<SHELLCMD> { pc.processShellCmd(id.image , val.image ,
overwrite); }
|
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Wed Jun
3 22:00:16 2009
@@ -1286,4 +1286,54 @@
log.info("Done");
}
+
+ /* Test case 32
+ * Test special values in characters
+ */
+ @Test
+ public void testComplexVals() throws Exception{
+ log.info("Starting test testComplexVals() ...");
+ try {
+ ParameterSubstitutionPreprocessor ps = new
ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir +
"/input5.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"loadfile =
/user/pig/tests/data/singlefile/textdoc.txt"};
+ String[] argFiles = {basedir+"/ConfFileComplexVal.txt"};
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir +
"/output1.pig");
+ pigExResultStream = new FileInputStream(basedir +
"/ExpectedResult3.txt");
+ BufferedReader inExpected = new BufferedReader(new
InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new
InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " +
"Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum
,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " +
"Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
+ } catch (ParseException e) {
+ fail ("Got ParseException : " + e.getMessage());
+ } catch (RuntimeException e) {
+ fail ("Got RuntimeException : " + e.getMessage());
+ } catch (Error e) {
+ fail ("Got error : " + e.getMessage());
+ }
+
+ log.info("Done");
+
+ }
}
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt?rev=781599&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt
(added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt Wed
Jun 3 22:00:16 2009
@@ -0,0 +1 @@
+storefile = /user/pig/tests/results/DefineClause_4.out
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt?rev=781599&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt Wed Jun
3 22:00:16 2009
@@ -0,0 +1,7 @@
+
+register /data/pigudf.jar;
+
+aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using
PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig?rev=781599&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig Wed Jun 3
22:00:16 2009
@@ -0,0 +1,8 @@
+%declare udfs /data/pigudf.jar
+
+register $udfs;
+
+aa = load '$loadfile ' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+store inactiveAccounts into '$storefile';
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig Wed Jun 3
22:00:16 2009
@@ -1,6 +1,7 @@
-register /home/y/lib/java/pigtest/testudf.jar;
-A = load '/user/pig/tests/data/singlefile/textdoc' using TextLoader();
-define X `perl -ne 'chomp $_; print "$_\n"'` output (stdout using
org.apache.pig.test.udf.storefunc.StringStore());
-B = stream A through X;
-store B into '/user/pig/tests/results/olgan.1209067990/DefineClause_4.out';
+register /data/pigudf.jar;
+
+aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using
PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';
