Author: gates
Date: Thu Jun 19 09:12:03 2008
New Revision: 669525

URL: http://svn.apache.org/viewvc?rev=669525&view=rev
Log:
PIG-269 First batch of merges from trunk to branch.  Thanks Pradeep.


Added:
    incubator/pig/branches/types/docs/
    incubator/pig/branches/types/docs/overview.html
    incubator/pig/branches/types/src/org/apache/pig/tools/parameters/
    
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj
    
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
    
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj
    
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestParamSubPreproc.java
    incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile1.txt
    incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile2.txt
    incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile3.txt
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileDuplicates.txt
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileSameParamMultipleTimes.txt
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithEmptyComments.txt
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithInvalidLines.txt
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithValidLines.txt
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult2.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/generate_date.sh
    incubator/pig/branches/types/test/org/apache/pig/test/data/input1.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/input2.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/input3.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/input4.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdlineParamPriortoDeclare.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdnameAsParamDeclare.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/inputDefault.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/inputEscape.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleDeclares.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleParams.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/inputNoVars.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubWithinLiteral.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinValue.pig
    
incubator/pig/branches/types/test/org/apache/pig/test/data/inputThreeParams.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/newinputS.txt
    incubator/pig/branches/types/test/org/apache/pig/test/data/output1.pig
    incubator/pig/branches/types/test/org/apache/pig/test/data/output3.pig
Modified:
    incubator/pig/branches/types/build.xml
    incubator/pig/branches/types/src/org/apache/pig/Main.java

Modified: incubator/pig/branches/types/build.xml
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=669525&r1=669524&r2=669525&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Thu Jun 19 09:12:03 2008
@@ -16,8 +16,7 @@
     <property name="src.dir" value="${basedir}/src/" />
     <property name="src.lib.dir" value="${basedir}/lib-src/" />
     <property name="src.gen.dir" value="${basedir}/src-gen/" />
-
-    <property name="javacc.home" value="${basedir}/lib" />
+    <property name="src.docs.dir" value="${basedir}/docs/" />
 
     <!-- javac properties -->
     <property name="javac.debug" value="on" />
@@ -25,9 +24,9 @@
     <property name="javac.deprecation" value="off" />
     <property name="javac.version" value="1.5" />
     <property name="javac.args" value="" />
-       <!-- TODO we should use warning...   <property 
name="javac.args.warnings" value="-Xlint:unchecked" /> -->
+    <!-- TODO we should use warning...   <property name="javac.args.warnings" 
value="-Xlint:unchecked" /> -->
     <property name="javac.args.warnings" value="" />
-    
+
     <!-- build properties -->
     <property name="build.dir" value="${basedir}/build" />
     <property name="build.classes" value="${build.dir}/classes" />
@@ -58,11 +57,17 @@
     <property name="junit.hadoop.conf" value="" />
     <property name="test.log.dir" value="${basedir}/test/logs"/>
     <property name="junit.hadoop.conf" value="${user.home}/pigtest/conf/"/>
-
+    <property name="test.output" value="no"/>
+       
     <!-- javacc properties -->
     <property name="src.gen.query.parser.dir" 
value="${src.gen.dir}/org/apache/pig/impl/logicalLayer/parser" />
     <property name="src.gen.script.parser.dir" 
value="${src.gen.dir}/org/apache/pig/tools/pigscript/parser" />
+    <property name="src.gen.param.parser.dir" 
value="${src.gen.dir}/org/apache/pig/tools/parameters" />
     <property name="src.gen.dot.parser.dir" 
value="${test.src.dir}/org/apache/pig/test/utils/dotGraph/parser" />
+    <property name="javacc.home" value="${basedir}/lib" />
+
+       <!-- javadoc properties  -->
+       <property name="javadoc.link.java" 
value="http://java.sun.com/j2se/1.5.0/docs/api/"; />
 
     <!-- ====================================================== -->
     <!-- Stuff needed by all targets                            -->
@@ -75,27 +80,52 @@
         <fileset file="${lib.dir}/junit-4.1.jar" />
         <fileset file="${lib.dir}/commons-collections-3.2.jar" />
     </path>
-    
+
     <path id="test.classpath">
-      <pathelement location="${build.classes}"/>
-      <pathelement location="${test.src.dir}"/>
-      <path refid="classpath"/>
+        <pathelement location="${build.classes}"/>
+        <pathelement location="${test.src.dir}"/>
+        <path refid="classpath"/>
     </path>
 
-        
+
     <target name="init">
         <mkdir dir="${src.gen.query.parser.dir}" />
         <mkdir dir="${src.gen.script.parser.dir}" />
+        <mkdir dir="${src.gen.param.parser.dir}" />
         <mkdir dir="${dist.dir}" />
         <mkdir dir="${build.classes}" />
         <mkdir dir="${test.build.classes}" />
         <mkdir dir="${src.gen.dot.parser.dir}" />
-    </target>
+        <tstamp>
+            <format property="timestamp" pattern="MMM dd yyyy, HH:mm:ss" />
+        </tstamp>
+        <svnversion outputproperty="svn.revision"/>
+    </target>
+
+    <macrodef name="svnversion">
+        <!-- the path needs to be small content otherwise it will take AGES ! 
-->
+        <attribute name="wcpath" default="${basedir}" />
+        <attribute name="outputproperty" />
+        <sequential>
+            <exec executable="svnversion" outputproperty="@{outputproperty}" 
failonerror="false" failifexecutionfails="false" >
+                <arg value="@{wcpath}" />
+                <redirector>
+                    <outputfilterchain>
+                        <tokenfilter>
+                            <!-- version can be xxxx, xxxx:yyyy, xxxxM, xxxxS 
or xxxx:yyyyMS , ... just get the working copy one -->
+                            <replaceregex pattern="((\d+).*)" replace="\2" />
+                        </tokenfilter>
+                    </outputfilterchain>
+                </redirector>
+            </exec>
+        </sequential>
+    </macrodef>
+
 
     <!-- ================================================================== -->
     <!-- Clean.  Delete the build files, and their directories              -->
     <!-- ================================================================== -->
-    <target name="clean">
+    <target name="clean" description="Cleanup build artifacts">
         <delete dir="${src.gen.dir}" />
         <delete dir="${build.dir}" />
         <delete dir="${src.gen.dot.parser.dir}" />
@@ -104,10 +134,12 @@
     <!-- ================================================================== -->
     <!-- Java Compiler Compiler, generate Parsers                           -->
     <!-- ================================================================== -->
-    <target name="cc-compile" depends="init">
+    <target name="cc-compile" depends="init" description="Create and Compile 
Parser">
         <jjtree 
target="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt" 
outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
         <javacc target="${src.gen.query.parser.dir}/QueryParser.jj" 
outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
         <javacc 
target="${src.dir}/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj" 
outputdirectory="${src.gen.script.parser.dir}" javacchome="${javacc.home}" />
+        <javacc 
target="${src.dir}/org/apache/pig/tools/parameters/PigFileParser.jj" 
outputdirectory="${src.gen.param.parser.dir}" javacchome="${javacc.home}" />
+        <javacc 
target="${src.dir}/org/apache/pig/tools/parameters/ParamLoader.jj" 
outputdirectory="${src.gen.param.parser.dir}" javacchome="${javacc.home}" />
         <jjtree 
target="${test.src.dir}/org/apache/pig/test/utils/dotGraph/Dot.jjt" 
outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
         <javacc target="${src.gen.dot.parser.dir}/Dot.jj" 
outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
     </target>
@@ -115,7 +147,7 @@
     <!-- ================================================================== -->
     <!-- Build sources                                          -->
     <!-- ================================================================== -->
-    <target name="compile" depends="init, cc-compile">
+    <target name="compile" depends="init, cc-compile" description="Compile all 
artifacts">
         <echo>*** Building Main Sources ***</echo>
         <antcall target="compile-sources">
             <param name="sources" 
value="${src.dir};${src.gen.dir};${src.lib.dir}/shock;${src.lib.dir}/bzip2" />
@@ -125,9 +157,6 @@
     </target>
 
     <target name="compile-test" depends="compile">
-       
-       
-        
         <echo>*** Building Test Sources ***</echo>
         <antcall target="compile-sources">
             <param name="sources" value="${test.src.dir}" />
@@ -149,14 +178,15 @@
     <!-- ================================================================== -->
     <!-- Documentation                                                      -->
     <!-- ================================================================== -->
-    <target name="javadoc" depends="init">
+    <target name="javadoc" depends="jar" description="Create documentation">
         <mkdir dir="${build.javadoc}" />
-        <javadoc overview="${src.dir}/overview.html" 
packagenames="org.apache.pig.*" destdir="${build.javadoc}" author="true" 
version="true" use="true" windowtitle="${Name} ${version} API" 
doctitle="${Name} ${version} API" bottom="Copyright &amp;copy; ${year} The 
Apache Software Foundation">
+        <javadoc overview="${src.docs.dir}/overview.html" 
packagenames="org.apache.pig.*" destdir="${build.javadoc}" author="true" 
version="true" use="true" windowtitle="${Name} ${version} API" 
doctitle="${Name} ${version} API" bottom="Copyright &amp;copy; ${year} The 
Apache Software Foundation">
             <packageset dir="${src.dir}" />
             <link href="${javadoc.link.java}" />
             <classpath>
                 <path refid="classpath" />
                 <pathelement path="${java.class.path}" />
+               <pathelement path="${output.jarfile}" />
             </classpath>
             <group title="pig" packages="org.apache.*" />
         </javadoc>
@@ -173,25 +203,47 @@
     <!-- ================================================================== -->
     <!-- TODO we should also exculte test here...                           -->
     <!-- ================================================================== -->
-    <target name="jar" depends="compile">
+    <target name="jar" depends="compile" description="Create pig jar">
+        <antcall target="jarWithSvn"/>
+        <antcall target="jarWithOutSvn"/>
+    </target>
+    
+    <target name="jarWithSvn" if="svn.revision">
+        <antcall target="buildJar">
+            <param name="svnString" value="${svn.revision}" />
+        </antcall>
+    </target>
+    
+    <target name="jarWithOutSvn" unless="svn.revision">
+        <antcall target="buildJar">
+            <param name="svnString" value=": unknown" />
+        </antcall>
+    </target>
+    
+    <target name="buildJar">
+        <echo>svnString ${svnString}</echo>
         <jar jarfile="${output.jarfile.core}" basedir="${build.classes}">
             <manifest>
                 <attribute name="Main-Class" value="org.apache.pig.Main" />
                 <section name="org/apache/pig">
+                    <attribute name="Implementation-Vendor" value="Apache" />
                     <attribute name="Implementation-Title" value="Pig" />
                     <attribute name="Implementation-Version" 
value="${version}" />
-                    <attribute name="Implementation-Vendor" value="Apache" />
+                    <attribute name="Build-TimeStamp" value="${timestamp}" />
+                    <attribute name="Svn-Revision" value="${svnString}" />
                 </section>
             </manifest>
         </jar>
-       <!-- @depricated -->
+        <!-- @depricated -->
         <jar jarfile="${output.jarfile}" basedir="${build.classes}">
             <manifest>
                 <attribute name="Main-Class" value="org.apache.pig.Main" />
                 <section name="org/apache/pig">
+                    <attribute name="Implementation-Vendor" value="Apache" />
                     <attribute name="Implementation-Title" value="Pig" />
                     <attribute name="Implementation-Version" 
value="${version}" />
-                    <attribute name="Implementation-Vendor" value="Apache" />
+                    <attribute name="Build-TimeStamp" value="${timestamp}" />
+                    <attribute name="Svn-Revision" value="${svnString}" />
                 </section>
             </manifest>
             <zipfileset src="${lib.dir}/junit-4.1.jar" />
@@ -221,9 +273,6 @@
             </classpath>
             <formatter type="${test.junit.output.format}" />
 
-            <batchtest fork="yes" todir="${test.log.dir}" if="testcase">
-                <fileset dir="test" includes="**/${testcase}.java"/>
-            </batchtest>
 
             <batchtest fork="yes" todir="${test.log.dir}" unless="testcase">
                 <fileset dir="test">
@@ -247,6 +296,9 @@
                     <exclude name="**/nightly/**" />
                 </fileset>
             </batchtest>
+            <batchtest fork="yes" todir="${test.log.dir}" if="testcase">
+                <fileset dir="test" includes="**/${testcase}.java"/>
+            </batchtest>
         </junit>
         <fail if="tests.failed">Tests failed!</fail>
     </target>
@@ -254,7 +306,7 @@
     <!-- ================================================================== -->
     <!-- D I S T R I B U T I O N                                            -->
     <!-- ================================================================== -->
-    <target name="package" depends="jar, javadoc">
+    <target name="package" depends="jar, javadoc" description="Create a Pig 
release">
         <mkdir dir="${dist.dir}" />
         <mkdir dir="${dist.dir}/lib" />
         <mkdir dir="${dist.dir}/scripts" />
@@ -298,7 +350,7 @@
     <!-- ================================================================== -->
     <!-- Make release tarball                                               -->
     <!-- ================================================================== -->
-    <target name="tar" depends="package">
+    <target name="tar" depends="package" description="Create release tarball">
         <tar compression="gzip" longfile="gnu" 
destfile="${build.dir}/${final.name}.tar.gz">
             <tarfileset dir="${dist.dir}" mode="664">
                 <exclude name="scripts/*" />

Added: incubator/pig/branches/types/docs/overview.html
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/docs/overview.html?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/docs/overview.html (added)
+++ incubator/pig/branches/types/docs/overview.html Thu Jun 19 09:12:03 2008
@@ -0,0 +1,13 @@
+<HTML>
+<BODY>
+Provides the classes necessary to create programs in the high-level Pig Latin
+language and the infrastructure for evaluating these programs. 
+<p>
+Pig is a platform for analyzing large data sets and consists of 2 layers:
+Pig's infrastructure layer consists of a compiler that produces sequences of 
+Map-Reduce programs, for which large-scale parallel implementations already 
+exist (e.g., the Hadoop project). Pig's language layer currently consists of 
+a textual language called Pig Latin for expressing data analysis programs.
+</BODY>
+</HTML>
+

Modified: incubator/pig/branches/types/src/org/apache/pig/Main.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/Main.java?rev=669525&r1=669524&r2=669525&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/Main.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/Main.java Thu Jun 19 
09:12:03 2008
@@ -35,10 +35,11 @@
 import org.apache.pig.ExecType;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
+import org.apache.pig.impl.util.JarManager;
 import org.apache.pig.tools.cmdline.CmdLineParser;
 import org.apache.pig.tools.grunt.Grunt;
 import org.apache.pig.tools.timer.PerformanceTimerFactory;
-
+import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
 
 public class Main
 {
@@ -64,6 +65,7 @@
 
     try {
         BufferedReader in = null;
+        BufferedReader pin = null;
         ExecMode mode = ExecMode.UNKNOWN;
         int port = 0;
         String file = null;
@@ -71,6 +73,10 @@
         boolean brief = false;
         String log4jconf = null;
         boolean verbose = false;
+        boolean debug = false;
+        boolean dryrun = false;
+        ArrayList<String> params = new ArrayList<String>();
+        ArrayList<String> paramFiles = new ArrayList<String>();
 
         CmdLineParser opts = new CmdLineParser(args);
         // Don't use -l, --latest, -c, --cluster, -cp, -classpath, -D as these
@@ -86,6 +92,10 @@
         opts.registerOpt('j', "jar", CmdLineParser.ValueExpected.REQUIRED);
         opts.registerOpt('v', "verbose", 
CmdLineParser.ValueExpected.NOT_ACCEPTED);
         opts.registerOpt('x', "exectype", 
CmdLineParser.ValueExpected.REQUIRED);
+        opts.registerOpt('i', "version", CmdLineParser.ValueExpected.OPTIONAL);
+        opts.registerOpt('p', "param", CmdLineParser.ValueExpected.OPTIONAL);
+        opts.registerOpt('m', "param_file", 
CmdLineParser.ValueExpected.OPTIONAL);
+        opts.registerOpt('r', "dryrun", 
CmdLineParser.ValueExpected.NOT_ACCEPTED);
 
         char opt;
         while ((opt = opts.getNextOpt()) != CmdLineParser.EndOfOpts) {
@@ -111,6 +121,7 @@
                       }
 
             case 'd':
+                debug = true;
                 logLevel = Level.toLevel(opts.getValStr(), Level.INFO);
                 break;
                 
@@ -137,6 +148,10 @@
                 break;
                       }
 
+            case 'm':
+                paramFiles.add(opts.getValStr());
+                break;
+                            
             case 'o': {
                    String gateway = System.getProperty("ssh.gateway");
                    if (gateway == null || gateway.length() == 0) {
@@ -147,6 +162,17 @@
                 break;
                       }
 
+            case 'p': 
+                String val = opts.getValStr();
+                params.add(opts.getValStr());
+                break;
+                            
+            case 'r': 
+                // currently only used for parameter substitition
+                // will be extended in the future
+                dryrun = true;
+                break;
+                            
             case 'v':
                 verbose = true;
                 break;
@@ -160,7 +186,9 @@
                    }
                    pigContext.setExecType(exectype);
                 break;
-
+            case 'i':
+               System.out.println(getVersionString());
+               return;
             default: {
                 Character cc = new Character(opt);
                 throw new AssertionError("Unhandled option " + cc.toString());
@@ -203,15 +231,26 @@
         // TODO Add a file appender for the logs
         // TODO Need to create a property in the properties file for it.
 
-        // Don't forget to undo all this for the port option.
-
-        // I might know what I want to do next, then again I might not.
+        // construct the parameter subsitution preprocessor
         Grunt grunt = null;
+        String substFile = null;
         switch (mode) {
         case FILE:
             // Run, using the provided file as a pig file
             in = new BufferedReader(new FileReader(file));
-            grunt = new Grunt(in, pigContext);
+
+            // run parameter substition preoprocessor first
+            substFile = file + ".substituted";
+            pin = runParamPreprocessor(in, params, paramFiles, substFile, 
debug || dryrun);
+            if (dryrun){
+                log.info("Dry run completed. Substitued pig script is at " + 
substFile);
+                return;
+            }
+
+            if (!debug)
+                new File(substFile).deleteOnExit();
+            
+            grunt = new Grunt(pin, pigContext);
             grunt.exec();
             return;
 
@@ -229,7 +268,7 @@
             grunt.exec();
             rc = 0;
             return;
-                     }
+            }
 
         default:
             break;
@@ -256,7 +295,19 @@
             }
             mode = ExecMode.FILE;
             in = new BufferedReader(new FileReader(remainders[0]));
-            grunt = new Grunt(in, pigContext);
+
+            // run parameter substition preoprocessor first
+            substFile = remainders[0] + ".substituted";
+            pin = runParamPreprocessor(in, params, paramFiles, substFile, 
debug || dryrun);
+            if (dryrun){
+                log.info("Dry run completed. Substitued pig script is at " + 
substFile);
+                return;
+            }
+
+            if (!debug)
+                new File(substFile).deleteOnExit();
+
+            grunt = new Grunt(pin, pigContext);
             grunt.exec();
             rc = 0;
             return;
@@ -276,9 +327,52 @@
         System.exit(rc);
     }
 }
+
+// retruns the stream of final pig script to be passed to Grunt
+private static BufferedReader runParamPreprocessor(BufferedReader 
origPigScript, ArrayList<String> params,
+                                            ArrayList<String> paramFiles, 
String scriptFile, boolean createFile) 
+                                throws 
org.apache.pig.tools.parameters.ParseException, IOException{
+    ParameterSubstitutionPreprocessor psp = new 
ParameterSubstitutionPreprocessor(50);
+    String[] type1 = new String[1];
+    String[] type2 = new String[1];
+
+    if (createFile){
+        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
+        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? 
params.toArray(type1) : null, 
+                                paramFiles.size() > 0 ? 
paramFiles.toArray(type2) : null);
+        return new BufferedReader(new FileReader (scriptFile));
+
+    } else {
+        StringWriter writer = new StringWriter();
+        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? 
params.toArray(type1) : null, 
+                                paramFiles.size() > 0 ? 
paramFiles.toArray(type2) : null);
+        return new BufferedReader(new StringReader(writer.toString()));
+    }
+}
     
+private static String getVersionString() {
+       String findContainingJar = JarManager.findContainingJar(Main.class);
+         try { 
+                 StringBuffer buffer = new  StringBuffer();
+          JarFile jar = new JarFile(findContainingJar); 
+          final Manifest manifest = jar.getManifest(); 
+          final Map <String,Attributes> attrs = manifest.getEntries(); 
+          Attributes attr = attrs.get("org/apache/pig");
+          String version = (String) attr.getValue("Implementation-Version");
+          String svnRevision = (String) attr.getValue("Svn-Revision");
+          String buildTime = (String) attr.getValue("Build-TimeStamp");
+          // we use a version string similar to svn 
+          //svn, version 1.4.4 (r25188)
+          // compiled Sep 23 2007, 22:32:34
+          return "Apache Pig version " + version + " (r" + svnRevision + ") 
\ncompiled "+buildTime;
+      } catch (Exception e) { 
+          throw new RuntimeException("unable to read pigs manifest file", e); 
+      } 
+}
+
 public static void usage()
 {
+       System.out.println("\n"+getVersionString()+"\n");
     System.out.println("USAGE: Pig [options] [-] : Run interactively in grunt 
shell.");
     System.out.println("       Pig [options] -e[xecute] cmd [cmd ...] : Run 
cmd(s).");
     System.out.println("       Pig [options] [-f[ile]] file : Run cmds found 
in file.");
@@ -292,5 +386,6 @@
     System.out.println("    -o, -hod read hod server from system property 
ssh.gateway");
     System.out.println("    -v, -verbose print all log messages to screen 
(default to print only INFO and above to screen)");
     System.out.println("    -x, -exectype local|mapreduce, mapreduce is 
default");
+    System.out.println("    -i, -version display version information");
 }
 }

Added: 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=669525&view=auto
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj 
(added)
+++ 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj 
Thu Jun 19 09:12:03 2008
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This grammar is used to process parameters passed from
+ * command line or config file
+ */
+
+options {
+    // Generate non-static functions
+    STATIC = false;
+}
+PARSER_BEGIN(ParamLoader)
+package org.apache.pig.tools.parameters;
+
+import java.io.IOException;
+import java.util.Hashtable;
+
+public class ParamLoader {
+
+    private PreprocessorContext pc;
+    public void setContext(PreprocessorContext pc) {
+        this.pc = pc;
+    }
+
+    private static String unquote(String s)
+    {
+        if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
+            return s.substring(1, s.length()-1);
+        else if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"')
+            return s.substring(1, s.length()-1);
+        else
+            return s;
+    }
+}
+
+PARSER_END(ParamLoader)
+
+SKIP : 
+{
+    "\n"
+|   "\r"
+|   " "
+|   "\t"
+}
+
+TOKEN:
+{
+    <#LETTER : ["a"-"z", "A"-"Z"] >
+    |
+    <#DIGIT : ["0"-"9"] >
+    |
+    <#SPECIALCHAR : ["_"] >
+}
+
+TOKEN :
+{
+//    <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> | 
<SPECIALCHAR>)*>
+    <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
+    |
+    <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " 
, "\n" , "\t" , "\r"] (~["\n","\r"])* >
+    |
+    <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* 
"'") >
+    |
+    <SHELLCMD: "`" (~["`"])* "`" >
+    |
+    <EQUALS: "="  >
+    |
+    <COMMENT: "#" (~["\n"])* ("\n"|"\r")  >
+
+}
+
+
+
+boolean Parse() throws IOException  :
+{
+    Token id, val;
+    boolean more=true;
+    String s;
+}
+{
+    (
+        (
+            id=<IDENTIFIER>
+            <EQUALS>
+            (
+                val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image);}
+                |
+                val=<ORD_LINE> {pc.processOrdLine(id.image , val.image);}
+                |
+                val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image); }
+                |
+                val=<LITERAL> { s = unquote(val.image); 
pc.processOrdLine(id.image , s); }
+            )
+        )
+        |
+        <COMMENT> { }
+        |
+        <EOF> { more=false; }
+    ) { return more; }
+
+
+}
+
+

Added: 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java?rev=669525&view=auto
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
 (added)
+++ 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
 Thu Jun 19 09:12:03 2008
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is the main driver for parameter substitution 
+ * http://wiki.apache.org/pig/ParameterSubstitution
+ */
+package org.apache.pig.tools.parameters;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
+
+import java.util.Hashtable;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.StringReader;
+import java.io.Writer;
+
+public class ParameterSubstitutionPreprocessor {
+
+    private PreprocessorContext pc;
+    private final Log log = LogFactory.getLog(getClass());
+    private ParamLoader paramParser;
+    private PigFileParser pigParser;
+
+    /**
+     * @param limit - max number of parameters to expect. Smaller values would
+     * would not cause incorrect behavior but would impact performance
+     */
+    public ParameterSubstitutionPreprocessor(int limit) {
+        pc = new PreprocessorContext(50);
+        StringReader sr = null;
+
+        paramParser  = new ParamLoader(sr);
+        paramParser.setContext(pc);
+
+        pigParser = new PigFileParser(sr);
+        pigParser.setContext(pc);
+    }
+
+    /**
+     * This is the main API that takes script template and produces pig script 
+     * @param pigInput - input stream that contains pig file
+     * @param pigOutput - stream where transformed file is written
+     * @param args - command line arguments in the order they appear on the 
command line; format: key=val
+     * @param argFiles - list of configuration files in the order they appear 
on the command line
+     */
+    public void genSubstitutedFile (BufferedReader pigInput, Writer pigOutput, 
String[] args, String[] argFiles) throws ParseException {
+
+        // load parameters from the files followed by parameters
+        // from command line both in the order they appear on the command line
+        // this enforces precedence rules 
+        if (argFiles!=null) {
+            for (int i=0;i<argFiles.length;i++) {
+                if (argFiles[i].length() > 0)
+                    loadParamsFromFile(argFiles[i]);
+            }
+        }
+        if (args!=null) {
+            for (int i=0;i<args.length;i++) {
+                if (args[i].length() > 0)
+                    loadParamsFromCmdline(args[i]);
+            }
+        }
+
+        // perform the substitution
+        parsePigFile(pigInput , pigOutput);
+    }
+
+    private void parsePigFile(BufferedReader in, Writer out) throws 
ParseException {
+
+        try {
+            String line;
+            int lineNum=0;
+
+            while((line = in.readLine())!= null)
+            {
+                lineNum++;
+
+                if (line.length() == 0) {
+                    out.append("\n");
+                    continue;
+                }
+
+                if ( !(line.startsWith("%")) ) {
+                    //process an ordinary pig line - perform substitution
+                    String sub_line = pc.substitute(line);
+                    out.append(sub_line);
+                    out.append("\n");
+                    continue;
+                }
+
+                //parse the declare/default line
+                pigParser.ReInit(new StringReader(line));
+                pigParser.Parse();
+            }
+
+            //close input and output streams
+            in.close();
+            out.flush();
+            out.close();
+        } catch (IOException e) {
+            RuntimeException rte = new RuntimeException(e.getMessage() , e);
+            throw rte;
+        }
+
+    }
+
+    /* 
+     * populates the param-val hashtable with parameters read from a file
+     * @param filename - name of the config file
+     */
+    private void loadParamsFromFile(String filename) throws ParseException {
+
+        try {
+            BufferedReader in = new BufferedReader(new FileReader(filename));
+            String line;
+ 
+            paramParser.ReInit(in);
+            while (paramParser.Parse()) {}
+            in.close();
+        } catch (IOException e) {
+            RuntimeException rte = new RuntimeException(e.getMessage() , e);
+            throw rte;
+        }
+
+    }
+
+    /*
+     * adds key-val pairs from cmd line to the param-val hashtable 
+     * @param param contains key-val of the form key='value'
+     */
+    private void loadParamsFromCmdline(String line) throws ParseException {
+        try {
+            // new lines are needed by the parser
+            paramParser.ReInit(new StringReader(line));
+            paramParser.Parse();
+        } catch (IOException e) {
+            RuntimeException rte = new RuntimeException(e.getMessage() , e);
+            throw rte;
+        }
+
+    }
+
+
+}

Added: 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=669525&view=auto
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj
 (added)
+++ 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj
 Thu Jun 19 09:12:03 2008
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This grammar is used to process %declare and %default commands
+ * within pig script
+ */
+
+options {
+    // Generate non-static functions
+    STATIC = false;
+       IGNORE_CASE = true;
+}
+PARSER_BEGIN(PigFileParser)
+package org.apache.pig.tools.parameters;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Hashtable;
+
+public class PigFileParser {
+    private PreprocessorContext pc;
+    public void setContext(PreprocessorContext pc) {
+        this.pc = pc;
+    }
+
+     private static String unquote(String s)
+     {
+        if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
+            return s.substring(1, s.length()-1);
+        else if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"')
+            return s.substring(1, s.length()-1);
+        else
+            return s;
+     }
+}
+
+PARSER_END(PigFileParser)
+
+SKIP : 
+{
+    "\n"
+|   "\r"
+|   " "
+|   "\t"
+}
+
+TOKEN:
+{
+    <#LETTER : ["a"-"z", "A"-"Z"] >
+    |
+    <#DIGIT : ["0"-"9"] >
+    |
+    <#SPECIALCHAR : ["_"] >
+}
+
+TOKEN :
+{
+    <DECLARE: "%declare" >
+    |
+    <PIGDEFAULT: "%default" >
+}
+
+TOKEN : 
+{
+    <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
+    |
+    <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* 
"'") >
+    |
+    <SHELLCMD: "`" (~["`"])* "`" >
+    |
+    <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " 
, "\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
+}
+
+
+
+void Parse() throws IOException  :
+{
+    Token id, val;
+    Boolean overwrite;
+    String s;
+}
+{
+
+    <DECLARE> { overwrite=true; }
+    (
+        id=<IDENTIFIER>
+        (
+            val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            |
+            val=<ORD_LINE> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            |
+            val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image, 
overwrite); }
+            |
+            val=<LITERAL> { s = unquote(val.image); 
pc.processOrdLine(id.image, s, overwrite); }
+
+        )
+    )
+    |
+    <PIGDEFAULT> { overwrite=false; }
+    (
+        id=<IDENTIFIER>
+        (
+            val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            |
+            val=<ORD_LINE> {pc.processOrdLine(id.image , val.image, 
overwrite);}
+            |
+            val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image , 
overwrite); }
+            |
+            val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image 
, s, overwrite); }
+        )
+    )   
+}

Added: 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=669525&view=auto
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java
 (added)
+++ 
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java
 Thu Jun 19 09:12:03 2008
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** 
+ * This is helper class for parameter substitution
+ */
+
+package org.apache.pig.tools.parameters;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Logger;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+import java.util.Hashtable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PreprocessorContext {
+
+    private Hashtable<String , String> param_val ;
+    
+    private final Log log = LogFactory.getLog(getClass());
+
+    /**
+     * @param limit - max number of parameters. Passing
+     *                smaller number only impacts performance
+     */         
+    public PreprocessorContext(int limit){
+        param_val = new Hashtable<String, String> (limit);
+    }
+    
+    /* 
+    public  void processLiteral(String key, String val) {
+        processLiteral(key, val, true);
+    } */
+
+    /**
+     * This method generates parameter value by running specified command
+     *
+     * @param key - parameter name
+     * @param val - string containing command to be executed
+     */
+    public  void processShellCmd(String key, String val) {
+        processShellCmd(key, val, true);
+    }
+
+    /**
+     * This method generates value for the specified key by
+     * performing substitution if needed within the value first.
+     *
+     * @param key - parameter name
+     * @param val - value supplied for the key
+     */
+    public  void processOrdLine(String key, String val) {
+        processOrdLine(key, val, true);
+    }
+
+    /*
+    public  void processLiteral(String key, String val, Boolean overwrite) {
+
+        if (param_val.containsKey(key)) {
+            if (overwrite) {
+                log.warn("Warning : Multiple values found for " + key + ". 
Using value " + val);
+            } else {
+                return;
+            }
+        }
+
+        String sub_val = substitute(val);
+        param_val.put(key, sub_val);
+    } */
+
+    /**
+     * This method generates parameter value by running specified command
+     *
+     * @param key - parameter name
+     * @param val - string containing command to be executed
+     */
+    public  void processShellCmd(String key, String val, Boolean overwrite) {
+
+        if (param_val.containsKey(key)) {
+            if (overwrite) {
+                log.warn("Warning : Multiple values found for " + key + ". 
Using value " + val);
+            } else {
+                return;
+            }
+        }
+
+        val=val.substring(1, val.length()-1); //to remove the backticks
+        String sub_val = substitute(val);
+        sub_val = executeShellCommand(sub_val);
+        param_val.put(key, sub_val);
+    } 
+
+    /**
+     * This method generates value for the specified key by
+     * performing substitution if needed within the value first.
+     *
+     * @param key - parameter name
+     * @param val - value supplied for the key
+     * @param overwide - specifies whether the value should be replaced if it 
already exists
+     */
+    public  void processOrdLine(String key, String val, Boolean overwrite) {
+
+        if (param_val.containsKey(key)) {
+            if (overwrite) {
+                log.warn("Warning : Multiple values found for " + key + ". 
Using value " + val);
+            } else {
+                return;
+            }
+        }
+
+        String sub_val = substitute(val);
+        param_val.put(key, sub_val);
+    } 
+
+
+    /*
+     * executes the 'cmd' in shell and returns result
+     */
+    private String executeShellCommand (String cmd) 
+    {
+        Process p;
+        String streamData="";
+        String streamError="";
+        try {
+            log.info("Executing command : " + cmd);
+            // we can't use exec directly since it does not handle
+            // case like foo -c "bar bar" correctly. It splits on white spaces 
even in presents of quotes
+            String[] cmdArgs = new String[3];
+            cmdArgs[0] = "bash";
+            cmdArgs[1] = "-c";
+            StringBuffer sb  = new StringBuffer("exec ");
+            sb.append(cmd);
+            cmdArgs[2] = sb.toString();
+
+            p = Runtime.getRuntime().exec(cmdArgs);
+
+        } catch (IOException e) {
+            RuntimeException rte = new RuntimeException("IO Exception while 
executing shell command : "+e.getMessage() , e);
+            throw rte;
+        }
+
+        int exitVal;
+        try {
+            exitVal = p.waitFor();
+        } catch (InterruptedException e) {
+            RuntimeException rte = new RuntimeException("Interrupted Thread 
Exception while waiting for command to get over"+e.getMessage() , e);
+            throw rte;
+        }
+
+        if (exitVal != 0) {
+            RuntimeException rte = new RuntimeException("Error executing shell 
command: " + cmd + ". Command exit with exit code of " + exitVal );
+            throw rte;
+        }
+
+        try{
+            InputStreamReader isr = new InputStreamReader(p.getInputStream());
+            BufferedReader br = new BufferedReader(isr);
+            String line=null;
+            while ( (line = br.readLine()) != null){
+                streamData+=(line+"\n");
+            }
+        } catch (IOException e){
+            RuntimeException rte = new RuntimeException("IO Exception while 
executing shell command : "+e.getMessage() , e);
+            throw rte;
+        }
+
+        try {
+            InputStreamReader isr = new InputStreamReader(p.getErrorStream());
+            BufferedReader br = new BufferedReader(isr);
+            String line=null;
+            while ( (line = br.readLine()) != null ) {
+                streamError += (line+"\n");
+            }
+            log.debug("Error stream while executing shell command : " + 
streamError);
+        } catch (Exception e) {
+            RuntimeException rte = new RuntimeException("IO Exception while 
executing shell command : "+e.getMessage() , e);
+            throw rte;
+        }
+
+        return streamData.trim();
+    }
+
+    private static String id_regex = "\\$[_]*[a-zA-Z][a-zA-Z_0-9]*";
+    public  String substitute(String line) {
+
+        int index = line.indexOf('$');
+        if (index == -1)       return line;
+
+        String replaced_line = new String(line);
+
+        Pattern identifier = Pattern.compile( id_regex );
+        Matcher keyMatcher = identifier.matcher( line );
+        String key="";
+        String val="";
+
+        while (keyMatcher.find()) {
+            // make sure that we don't perform parameter substitution
+            // for escaped vars of the form \$<id>
+            if ( (keyMatcher.start() == 0) || (line.charAt( keyMatcher.start() 
- 1)) != '\\' ) {
+                key = keyMatcher.group().substring(1);         //skip the '$'
+                if (!(param_val.containsKey(key))) {
+                    throw new RuntimeException("Undefined parameter : "+key);
+                }
+                val = param_val.get(key);
+                //String litVal = Matcher.quoteReplacement(val);
+                replaced_line = replaced_line.replaceFirst("\\$"+key, val); 
+            }
+        }
+
+        // unescape $<id>
+        replaced_line = replaced_line.replaceAll("\\\\\\$","\\$");
+        return replaced_line;
+    }
+
+}
+
+


Reply via email to