Author: olga
Date: Tue Nov 24 17:10:47 2009
New Revision: 883790

URL: http://svn.apache.org/viewvc?rev=883790&view=rev
Log:
PIG-598: Parameter substitution () should not be performed in
comments (thejas via olgan)

Added:
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig
    
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
    
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
    hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
    
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig
    
hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue Nov 24 17:10:47 2009
@@ -36,6 +36,9 @@
 
 BUG FIXES
 
+PIG-598: Parameter substitution ($PARAMETER) should not be performed in
+comments (thejas via olgan)
+
 PIG-1064: Behaviour of COGROUP with and without schema when using "*" operator
 (pradeepkth)
 

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj 
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj Tue Nov 
24 17:10:47 2009
@@ -32,6 +32,11 @@
 import java.io.IOException;
 import java.util.Hashtable;
 
+//warnings in by code generated by javacc cannot be fixed here, 
+// so suppressing all warnings for this class. But this does not help in 
+//supressing Warnings in other classes generated by this .jj file
+...@suppresswarnings("all")
+
 public class ParamLoader {
 
     private PreprocessorContext pc;

Modified: 
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
 (original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
 Tue Nov 24 17:10:47 2009
@@ -52,7 +52,6 @@
 
         paramParser  = new ParamLoader(sr);
         paramParser.setContext(pc);
-
         pigParser = new PigFileParser(sr);
         pigParser.setContext(pc);
     }
@@ -87,33 +86,10 @@
     }
 
     private void parsePigFile(BufferedReader in, Writer out) throws 
ParseException {
-
+        pigParser.setOutputWriter(out);
+        pigParser.ReInit(in);
         try {
-            String line;
-            int lineNum=0;
-
-            while((line = in.readLine())!= null)
-            {
-                lineNum++;
-
-                if (line.length() == 0) {
-                    out.append("\n");
-                    continue;
-                }
-
-                if ( !(line.startsWith("%")) ) {
-                    //process an ordinary pig line - perform substitution
-                    String sub_line = pc.substitute(line);
-                    out.append(sub_line);
-                    out.append("\n");
-                    continue;
-                }
-
-                //parse the declare/default line
-                pigParser.ReInit(new StringReader(line));
-                pigParser.Parse();
-            }
-
+            pigParser.Parse();
             //close input and output streams
             in.close();
             out.flush();
@@ -122,7 +98,6 @@
             RuntimeException rte = new RuntimeException(e.getMessage() , e);
             throw rte;
         }
-
     }
 
     /* 

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj 
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Tue 
Nov 24 17:10:47 2009
@@ -18,7 +18,7 @@
 
 /**
  * This grammar is used to process %declare and %default commands
- * within pig script
+ * within pig script, and substitute parameter values
  */
 
 options {
@@ -26,20 +26,34 @@
     STATIC = false;
        IGNORE_CASE = true;
     JAVA_UNICODE_ESCAPE = true;
+    //DEBUG_PARSER = true;
+    LOOKAHEAD = 2;          
+    
 }
 PARSER_BEGIN(PigFileParser)
+
 package org.apache.pig.tools.parameters;
 
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Hashtable;
+import java.io.Writer;
+import java.lang.StringBuilder;
+
+//warnings in by code generated by javacc cannot be fixed here, 
+// so suppressing all warnings for this class. But this does not help in 
+//supressing Warnings in other classes generated by this .jj file
+...@suppresswarnings("all")
 
 public class PigFileParser {
     private PreprocessorContext pc;
+    private Writer out;
     public void setContext(PreprocessorContext pc) {
         this.pc = pc;
     }
-
+    public void setOutputWriter(Writer out) {
+        this.out = out;
+    }
      private static String unquote(String s)
      {
         if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
@@ -53,12 +67,20 @@
 
 PARSER_END(PigFileParser)
 
-SKIP : 
+TOKEN : 
 {
-    "\n"
-|   "\r"
-|   " "
-|   "\t"
+    <NEWLINE: "\n" | "\r">
+    |    
+    <SPACE: " " | "\t">
+}
+
+// comments(single line and multi-line)
+TOKEN : 
+{
+   <COMMENT: "--"(~["\r","\n"])*
+          |  "#!" (~["\r","\n"])*
+          | "/*" (~["*"])* "*" ("*" | (~["*","/"] (~["*"])* "*"))* "/"
+   >   
 }
 
 TOKEN:
@@ -68,13 +90,15 @@
     <#DIGIT : ["0"-"9"] >
     |
     <#SPECIALCHAR : ["_"] >
+    |
+    <#DOLLAR : ["$"]>
 }
 
 TOKEN :
 {
     <DECLARE: "%declare" >
     |
-    <PIGDEFAULT: "%default" >
+    <PIGDEFAULT: "%default" > 
 }
 
 TOKEN : 
@@ -85,30 +109,75 @@
     |
     <SHELLCMD: "`" (~["`"])* "`" >
     |
-    <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , 
"\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
+    // see others() rule for use of OTHER and NOT_OTHER_CHAR
+    // others() is supposed to match 'everything else'. To ensure that others()
+    // don't swallow other(all the ones above) tokens, it uses two tokens 
OTHER and NOT_OTHER_CHAR
+    // NOT_OTHER_CHAR consists of first characters of other tokens, and OTHER 
consists of one
+    // or more chars that don't belong to NOT_OTHER_CHAR. Since the tokeniser 
matches the longest 
+    // match, other tokens will get matched instead of NOT_OTHER_CHAR.
+    <OTHER: (~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , 
"\n" , "\t" , "\r", "%", "/", "-", "$"])+ >
+    |
+    <NOT_OTHER_CHAR: ["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , 
" " , "\n" , "\t" , "\r", "%", "/", "-", "$"] >
+ 
 }
 
+void Parse() throws IOException : {}
+{
+    (input())*<EOF>
+}
 
-
-void Parse() throws IOException  :
+void input() throws IOException  :
 {
-    Token id, val;
-    Boolean overwrite;
     String s;
+    Token strTok;
 }
 {
 
-    <DECLARE> { overwrite=true; }
+    <DECLARE> 
     (
-        id=<IDENTIFIER>
+        param_value(true) // overwrite=true
+    )
+    |
+    <PIGDEFAULT>
+    (
+        param_value(false) // overwrite=false
+    )
+    |
+    s = paramString(){}
+    {
+        //process an ordinary pig line - perform substitution
+        String sub_line = pc.substitute(s);
+        out.append(sub_line );
+    }
+    |
+    strTok = <IDENTIFIER>{}
+    {
+        out.append(strTok.image );
+    }
+    |
+    write_ignore_toks()
+
+}
+
+void param_value(boolean overwrite) throws IOException:
+{
+    Token id, val;
+    String s;
+    String other;
+    Token strTok;
+}
+{
+    (ignore_toks_nonewline())*         
+    id=<IDENTIFIER>
+    (ignore_toks_nonewline())*               
         (
-            val=<OTHER> { pc.processOrdLine(id.image , val.image);}
+            s=others() (ignore_toks_nonewline())* write_newline() { 
pc.processOrdLine(id.image , s, overwrite);}
             |
-            val=<IDENTIFIER>
+            val=<IDENTIFIER> // this construct is for cases like a.123
             {s = val.image;}
-            ( 
-                val =<OTHER>
-                {s += val.image;}
+            (   LOOKAHEAD( 2 )
+                other = others() (ignore_toks_nonewline())* write_newline()
+                { s += other; }
             )?
             {pc.processOrdLine(id.image , s, overwrite);}
             |
@@ -117,25 +186,136 @@
             val=<LITERAL> { s = unquote(val.image); 
pc.processOrdLine(id.image, s, overwrite); }
 
         )
-    )
-    |
-    <PIGDEFAULT> { overwrite=false; }
+}
+
+//match others, see comments above 
+//on description of OTHER , NOT_OTHER_CHAR
+String others() throws IOException : 
+{ Token t;
+  StringBuilder sb = new StringBuilder();  
+}
+{
     (
-        id=<IDENTIFIER>
-        (
-            val=<OTHER> { pc.processOrdLine(id.image , val.image);}
-            |
-            val=<IDENTIFIER>
-            {s = val.image;}
-            ( 
-                val =<OTHER>
-                {s += val.image;}
-            )?
-            {pc.processOrdLine(id.image , s, overwrite);}
-            |
-            val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image , 
overwrite); }
-            |
-            val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image 
, s, overwrite); }
-        )
+        t=<OTHER>
+        {
+            sb.append(t.image);
+        }
+        |
+        t=<NOT_OTHER_CHAR>
+        {
+            sb.append(t.image);
+        } 
     )   
+    (   
+        t=<OTHER>
+        { 
+            sb.append(t.image);
+        }   
+        |
+        t=<NOT_OTHER_CHAR>
+        {
+            sb.append(t.image);
+        }   
+        |
+        t=<IDENTIFIER>
+        {
+        //eg of this match is a unquoted filename - /d1/abc/
+            sb.append(t.image);
+        }   
+        |
+        t=<LITERAL>
+        {
+            sb.append(t.image);
+        }  
+        |
+        t=<SHELLCMD>
+        {
+            sb.append(t.image);
+        }   
+    )*{
+        return sb.toString();
+    }
+}
+
+
+// a string that can contain parameter
+String paramString() throws IOException : 
+{Token t;
+ String str;
+}
+{
+    str = others()
+    {
+        return str;
+    }    
+    |
+    t=<LITERAL>
+    {
+        return t.image;
+    }
+    |
+    t = <SHELLCMD>{}
+    {
+        return t.image;
+    }    
+}
+
+// write the newlines,spaces and comments to preserve formatting
+void write_ignore_toks() throws IOException : 
+{ String str;}
+{
+    str = ignore_toks(){
+        out.append(str);
+    }
+}
+
+
+// match the newlines,spaces and comments
+String ignore_toks_nonewline() throws IOException : 
+{   Token t; }
+{
+    t = <COMMENT>{
+        return t.image;
+    }
+    |
+    t = <SPACE>{
+        return t.image;
+    }
+}
+
+
+// match the newlines,spaces and comments
+String ignore_toks() throws IOException : 
+{   Token t;
+    String str;
 }
+{
+    t = <COMMENT>{
+        return t.image;
+    }
+    |
+    str = space_or_newline(){
+        return str;
+    }
+}
+
+String space_or_newline() :
+{ Token t; }
+{
+    t = <SPACE>{
+        return t.image;
+    }
+    |
+    t = <NEWLINE>{
+        return t.image;
+    }
+}
+
+void write_newline() throws IOException:
+{ Token t; }
+{
+    t = <NEWLINE>{
+        out.append(t.image);
+    }
+}
+

Modified: 
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java 
(original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java 
Tue Nov 24 17:10:47 2009
@@ -210,16 +210,16 @@
         return streamData.trim();
     }
 
-    private static String id_regex = "\\$[_]*[a-zA-Z][a-zA-Z_0-9]*";
+    private Pattern id_pattern = 
Pattern.compile("\\$[_]*[a-zA-Z][a-zA-Z_0-9]*");
+    
     public  String substitute(String line) {
 
         int index = line.indexOf('$');
         if (index == -1)       return line;
 
         String replaced_line = line;
-
-        Pattern identifier = Pattern.compile( id_regex );
-        Matcher keyMatcher = identifier.matcher( line );
+        
+        Matcher keyMatcher = id_pattern.matcher( line );
         String key="";
         String val="";
 

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java 
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Tue Nov 
24 17:10:47 2009
@@ -167,7 +167,7 @@
             ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
 
             FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
-            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResultDefault.pig");
             BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
             BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
 
@@ -259,7 +259,7 @@
             ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
 
             FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
-            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult4.pig");
             BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
             BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
 
@@ -310,7 +310,7 @@
             ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
 
             FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
-            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult4.pig");
             BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
             BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
 
@@ -359,7 +359,7 @@
             ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
 
             FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
-            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResultCmdLnPriorDeclare.pig");
             BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
             BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
 
@@ -409,7 +409,7 @@
             ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
 
             FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
-            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult4.pig");
             BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
             BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
 
@@ -913,7 +913,7 @@
             ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
 
             FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
-            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResult.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResultMulDecs.pig");
             BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
             BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
 
@@ -1336,4 +1336,51 @@
         log.info("Done");
 
     }
+    
+    /* Test case 25
+     *   Test that params in comments are untouched
+     */
+    @Test
+    public void testCommentWithParam() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new 
ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + 
"/inputComment.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + 
"/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + 
"/ExpectedResultComment.pig");
+            BufferedReader inExpected = new BufferedReader(new 
InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new 
InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in 
line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line 
num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
 }

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig Tue Nov 
24 17:10:47 2009
@@ -8,4 +8,4 @@
 store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
 grpInactiveAcct = group inactiveAccounts all;
 countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
-store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig 
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig Tue Nov 
24 17:10:47 2009
@@ -8,4 +8,4 @@
 store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
 grpInactiveAcct = group inactiveAccounts all;
 countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
-store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig Tue Nov 
24 17:10:47 2009
@@ -0,0 +1,14 @@
+
+
+
+
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: 
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig?rev=883790&view=auto
==============================================================================
--- 
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
 (added)
+++ 
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
 Tue Nov 24 17:10:47 2009
@@ -0,0 +1,12 @@
+
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
+
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig 
(added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig 
Tue Nov 24 17:10:47 2009
@@ -0,0 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
+
+/* comment */ 
+
+/* l = load '$zxv';
+   store l into '$zxcv';
+*/
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate /* $col 1 */ $0,$12,$7;  
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- 
testing $comment
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig 
(added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig 
Tue Nov 24 17:10:47 2009
@@ -0,0 +1,12 @@
+
+
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig 
(added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig 
Tue Nov 24 17:10:47 2009
@@ -0,0 +1,13 @@
+
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig Tue Nov 24 
17:10:47 2009
@@ -1,5 +1,4 @@
 %declare udfs /data/pigudf.jar
-
 register $udfs;
 
 aa = load '$loadfile ' using PigStorage('\x01');

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig Tue Nov 24 
17:10:47 2009
@@ -0,0 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
+%default date /* comment */ 1000 -- setting default
+/* comment */ %declare arity 16
+
+/* l = load '$zxv';
+   store l into '$zxcv';
+*/
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using 
PigStorage('\x01');
+bb = filter aa by (ARITY == '$arity') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
+a = foreach bb generate /* $col 1 */ $0,$12,$7;  
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- 
testing $comment
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig Tue Nov 24 
17:10:47 2009
@@ -1,5 +1,4 @@
 %default date '20080228'
-
 aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using 
PigStorage('\x01');
 bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
 a = foreach bb generate $0,$12,$7;

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig Tue Nov 24 
17:10:47 2009
@@ -1,5 +1,4 @@
 %default column1 'asdf'
-
 aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
 bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
 a = foreach bb generate $0,$12,$7;

Modified: 
hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- 
hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
 (original)
+++ 
hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
 Tue Nov 24 17:10:47 2009
@@ -1,4 +1,5 @@
 
+
 %DECLARE date '20080228'
 %DECLARE formatted_date `sh test/org/apache/pig/test/data/generate_date.sh 
$date`
 aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$formatted_date'
 using PigStorage('\x01');

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig 
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig Tue Nov 
24 17:10:47 2009
@@ -1,9 +1,7 @@
 %default tableName 'value to be skipped'
-
 aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using 
PigStorage('\x01');
 bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
 a = foreach bb generate $0,$12,$7;
-
 %declare tableName 'inactiveAccounts'
 --generate inactive accts
 $tableName = filter a by ($1 neq '') and ($1 == '2') parallel 400;

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig Tue Nov 24 
17:10:47 2009
@@ -1,7 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
 
-register /data/pigudf.jar;
+/* comment */ 
 
-aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using 
PigStorage('\x01');
+/* l = load '$zxv';
+   store l into '$zxcv';
+*/
+aa = load 
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' 
using PigStorage('\x01');
 bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 
'ss') parallel 400;
-a = foreach bb generate $0,$12,$7;
-store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';
+a = foreach bb generate /* $col 1 */ $0,$12,$7;  
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- 
testing $comment
+store inactiveAccounts into 
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts 
); }
+store countInactiveAcct into 
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';


Reply via email to