Author: olga
Date: Wed Jun  3 22:00:16 2009
New Revision: 781599

URL: http://svn.apache.org/viewvc?rev=781599&view=rev
Log:
PIG-564: problem with parameter substitution and special charachters (olgan)

Added:
    hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt
    hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
    hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
    hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
    hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Jun  3 22:00:16 2009
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+
+
 Pig Change Log
 
 Trunk (unreleased changes)
@@ -26,6 +28,8 @@
 
 PIG-825: PIG_HADOOP_VERSION should be set to 18 (dvryaboy via gates).
 
+PIG-795: support for SAMPLE command (ericg via olgan)
+
 PIG-619: Create one InputSplit even when the input file is zero length
          so that hadoop runs maps and creates output for the next
                 job (gates).
@@ -37,19 +41,27 @@
 PIG-712: Added utility functions to create schemas for tuples and bags (zjffdu
 via gates).
 
+PIG-652: Adapt changes in store interface to multi-query changes (hagleitn via 
gates).
+
 PIG-775: PORelationToExprProject should create a NonSpillableDataBag to create
 empty bags (pradeepkth)
 
 PIG-741: Allow limit to be nested in a foreach.
 
+PIG-627: multiquery support phase 3 (hagleitn and Richard Ding via olgan)
+
 PIG-743: To implement clover (gkesavan)
 
 PIG-701: Implement IVY for resolving pig dependencies (gkesavan)
 
 PIG-626: Add access to hadoop counters (shubhamc via gates).
 
+PIG-627: multiquery support phase 1 and phase 2 (hagleitn and Richard Ding via 
pradeepkth)
+
 BUG FIXES
 
+PIG-564: problem with parameter substitution and special charachters (olgan)
+
 PIG-802: PERFORMANCE: not creating bags for ORDER BY (serakesh via olgan)
 
 PIG-816: PigStorage() does not accept Unicode characters in its contructor 
(pradeepkth)
@@ -650,10 +662,3 @@
 
     PIG-284: target for building source jar (oae via olgan)
 
-    PIG-627: multiquery support phase 1 and phase 2 (hagleitn and Richard Ding 
via pradeepkth)
-
-    PIG-652: Adapt changes in store interface to multi-query changes (hagleitn 
via gates).
-
-    PIG-627: multiquery support phase 3 (hagleitn and Richard Ding via olgan)
-
-    PIG-795: support for SAMPLE command (ericg via olgan)

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj 
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj Wed Jun 
 3 22:00:16 2009
@@ -74,7 +74,7 @@
 //    <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> | 
<SPECIALCHAR>)*>
     <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
     |
-    <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " 
, "\n" , "\t" , "\r"] (~["\n","\r"])* >
+    <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , 
"\n" , "\t" , "\r"] (~["\n","\r"])* >
     |
     <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* 
"'") >
     |
@@ -100,11 +100,17 @@
             id=<IDENTIFIER>
             <EQUALS>
             (
-                val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image);}
+                val=<OTHER> { pc.processOrdLine(id.image , val.image);}
                 |
-                val=<ORD_LINE> {pc.processOrdLine(id.image , val.image);}
+                val=<IDENTIFIER> 
+                {s = val.image;}
+                (
+                    val =<OTHER>
+                    {s += val.image;}
+                )?
+                {pc.processOrdLine(id.image , s);}
                 |
-                val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image); }
+                val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image);}
                 |
                 val=<LITERAL> { s = unquote(val.image); 
pc.processOrdLine(id.image , s); }
             )

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj 
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Wed 
Jun  3 22:00:16 2009
@@ -85,7 +85,7 @@
     |
     <SHELLCMD: "`" (~["`"])* "`" >
     |
-    <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " 
, "\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
+    <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , 
"\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
 }
 
 
@@ -102,9 +102,15 @@
     (
         id=<IDENTIFIER>
         (
-            val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            val=<OTHER> { pc.processOrdLine(id.image , val.image);}
             |
-            val=<ORD_LINE> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            val=<IDENTIFIER>
+            {s = val.image;}
+            ( 
+                val =<OTHER>
+                {s += val.image;}
+            )?
+            {pc.processOrdLine(id.image , s, overwrite);}
             |
             val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image, 
overwrite); }
             |
@@ -117,9 +123,15 @@
     (
         id=<IDENTIFIER>
         (
-            val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            val=<OTHER> { pc.processOrdLine(id.image , val.image);}
             |
-            val=<ORD_LINE> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            val=<IDENTIFIER>
+            {s = val.image;}
+            ( 
+                val =<OTHER>
+                {s += val.image;}
+            )?
+            {pc.processOrdLine(id.image , s, overwrite);}
             |
             val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image , 
overwrite); }
             |

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java 
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Wed Jun  
3 22:00:16 2009
@@ -1286,4 +1286,54 @@
         log.info("Done");
 
     }
+
+    /* Test case 32   
+     * Test special values in characters
+     */
+    @Test
+    public void testComplexVals() throws Exception{
+        log.info("Starting test testComplexVals() ...");
+        try {
+            ParameterSubstitutionPreprocessor ps = new 
ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + 
"/input5.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"loadfile = 
/user/pig/tests/data/singlefile/textdoc.txt"};
+            String[] argFiles = {basedir+"/ConfFileComplexVal.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult3.txt");
+            BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Command line parameter substitution failed. " + 
"Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum 
,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Command line parameter substitution failed. " + 
"Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+
+    }
 }

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt?rev=781599&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt 
(added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ConfFileComplexVal.txt Wed 
Jun  3 22:00:16 2009
@@ -0,0 +1 @@
+storefile = /user/pig/tests/results/DefineClause_4.out

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt?rev=781599&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult3.txt Wed Jun  
3 22:00:16 2009
@@ -0,0 +1,7 @@
+
+register /data/pigudf.jar;
+
+aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using 
PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig?rev=781599&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig Wed Jun  3 
22:00:16 2009
@@ -0,0 +1,8 @@
+%declare udfs /data/pigudf.jar
+
+register $udfs;
+
+aa = load '$loadfile ' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+store inactiveAccounts into '$storefile';

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=781599&r1=781598&r2=781599&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig Wed Jun  3 
22:00:16 2009
@@ -1,6 +1,7 @@
 
-register /home/y/lib/java/pigtest/testudf.jar;
-A = load '/user/pig/tests/data/singlefile/textdoc' using TextLoader();
-define X `perl -ne 'chomp $_; print "$_\n"'` output (stdout using 
org.apache.pig.test.udf.storefunc.StringStore());
-B = stream A through X;
-store B into '/user/pig/tests/results/olgan.1209067990/DefineClause_4.out';
+register /data/pigudf.jar;
+
+aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using 
PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';


Reply via email to