Modified: incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original) +++ incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu Sep 4 14:25:41 2008 @@ -188,6 +188,7 @@ <PIG_START> MORE : { <"'"> {prevState = PIG_START;} : IN_STRING +| <"`"> {prevState = PIG_START;} : IN_COMMAND | <["A","a"]["S","s"]> {prevState = PIG_START;} : SCHEMA_DEFINITION | <["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"]> {prevState = PIG_START;} : GENERATE | <"{"> {pigBlockLevel = 1;} : IN_BLOCK @@ -220,6 +221,14 @@ | <(~[])> } +<IN_COMMAND> MORE : +{ + <"\\`"> +| <"`"> { SwitchTo(prevState);} +| <("\n" | "\r" | "\r\n")> {secondary_prompt();} +| <(~[])> +} + <GENERATE> MORE : { <"{"> @@ -282,6 +291,7 @@ | <"{"> {pigBlockLevel++;} | <"}"(";")?> {pigBlockLevel--; if (pigBlockLevel == 0) SwitchTo(PIG_END);} | <"'"> {prevState = IN_BLOCK;} : IN_STRING +| <"`"> {prevState = IN_BLOCK;} : IN_COMMAND | <"--"> {prevState = IN_BLOCK;} : SINGLE_LINE_COMMENT | <"/*"> {prevState = IN_BLOCK;} : MULTI_LINE_COMMENT | <("\n" | "\r" | "\r\n")> {secondary_prompt();}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java Thu Sep 4 14:25:41 2008 @@ -115,7 +115,7 @@ new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ; FileSpec filespec2 = new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")); - LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null) ; + LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, true) ; LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ; plan.add(load) ; Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java Thu Sep 4 14:25:41 2008 @@ -153,7 +153,7 @@ FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName())); FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName())); - POLoad ld = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true); POStore st = new POStore(new OperatorKey("", r.nextLong())); ld.setPc(pc); ld.setLFile(LFSpec); @@ -200,7 +200,7 @@ FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName())); FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName())); - POLoad ld = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true); POStore st = new POStore(new OperatorKey("", r.nextLong())); ld.setPc(pc); ld.setLFile(LFSpec); @@ -247,7 +247,7 @@ FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName())); FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName())); - POLoad ld = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true); POStore st = new POStore(new OperatorKey("", r.nextLong())); ld.setPc(pc); ld.setLFile(LFSpec); @@ -309,7 +309,7 @@ FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName())); FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName())); - POLoad ld = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true); POStore st = new POStore(new OperatorKey("", r.nextLong())); ld.setPc(pc); ld.setLFile(LFSpec); @@ -376,7 +376,7 @@ FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName(), new String[]{","})); FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName())); - POLoad ld = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true); POStore st = new POStore(new OperatorKey("", r.nextLong())); ld.setPc(pc); ld.setLFile(LFSpec); Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java Thu Sep 4 14:25:41 2008 @@ -373,7 +373,7 @@ FileSpec LFSpec = new FileSpec(ldFile, new FuncSpec(PigStorage.class.getName(), new String[]{","})); FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName())); - POLoad ld = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true); POStore st = new POStore(new OperatorKey("", r.nextLong())); ld.setPc(pc); ld.setLFile(LFSpec); Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java Thu Sep 4 14:25:41 2008 @@ -896,11 +896,11 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; LOLoad load2 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1020,11 +1020,11 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; LOLoad load2 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1192,7 +1192,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1260,7 +1260,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1330,7 +1330,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1400,7 +1400,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1480,7 +1480,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1578,7 +1578,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1673,7 +1673,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1801,7 +1801,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -1905,11 +1905,11 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; LOLoad load2 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -2076,11 +2076,11 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; LOLoad load2 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; @@ -2215,11 +2215,11 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; LOLoad load2 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // schema for input#1 Schema inputSchema1 = null ; Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java Thu Sep 4 14:25:41 2008 @@ -445,7 +445,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // set schemas load1.setEnforcedSchema(null) ; @@ -511,7 +511,7 @@ LOLoad load1 = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; // set schemas load1.setEnforcedSchema(null) ; Modified: incubator/pig/branches/types/test/org/apache/pig/test/Util.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/Util.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/Util.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/Util.java Thu Sep 4 14:25:41 2008 @@ -17,9 +17,14 @@ */ package org.apache.pig.test; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import static java.util.regex.Matcher.quoteReplacement; +import junit.framework.Assert; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.data.*; @@ -145,6 +150,45 @@ return t; } + /** + * Helper to create a temporary file with given input data for use in test cases. + * + * @param tmpFilenamePrefix file-name prefix + * @param tmpFilenameSuffix file-name suffix + * @param inputData input for test cases, each string in inputData[] is written + * on one line + * @return [EMAIL PROTECTED] File} handle to the created temporary file + * @throws IOException + */ + static public File createInputFile(String tmpFilenamePrefix, + String tmpFilenameSuffix, + String[] inputData) + throws IOException { + File f = File.createTempFile(tmpFilenamePrefix, tmpFilenameSuffix); + PrintWriter pw = new PrintWriter(f); + for (int i=0; i<inputData.length; i++){ + pw.println(inputData[i]); + } + pw.close(); + return f; + } + + /** + * Helper function to check if the result of a Pig Query is in line with + * expected results. + * + * @param actualResults Result of the executed Pig query + * @param expectedResults Expected results to validate against + */ + static public void checkQueryOutputs(Iterator<Tuple> actualResults, + Tuple[] expectedResults) { + + for (Tuple expected : expectedResults) { + Tuple actual = actualResults.next(); + Assert.assertEquals(expected.toString(), actual.toString()); + } + } + static public void printQueryOutput(Iterator<Tuple> actualResults, Tuple[] expectedResults) { @@ -160,4 +204,18 @@ } System.out.println("---End----") ; } + + /** + * Helper method to replace all occurrences of "\" with "\\" in a + * string. This is useful to fix the file path string on Windows + * where "\" is used as the path separator. + * + * @param str Any string + * @return The resulting string + */ + public static String encodeEscape(String str) { + String regex = "\\\\"; + String replacement = quoteReplacement("\\\\"); + return str.replaceAll(regex, replacement); + } } Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java Thu Sep 4 14:25:41 2008 @@ -568,7 +568,7 @@ public static POLoad topLoadOp() { - POLoad ret = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ret = new POLoad(new OperatorKey("", r.nextLong()), true); ret.setPc(pc); return ret; } Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java Thu Sep 4 14:25:41 2008 @@ -170,11 +170,11 @@ public static boolean areFilesSame(FileSpec expLocal, FileSpec actHadoop, PigContext pc) throws ExecException, IOException{ Random r = new Random(); - POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()), true); ldExp.setPc(pc); ldExp.setLFile(expLocal); - POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong())); + POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()), true); ldAct.setPc(pc); ldAct.setLFile(actHadoop); Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java Thu Sep 4 14:25:41 2008 @@ -42,7 +42,7 @@ LOLoad load = new LOLoad(plan, genNewOperatorKey(), new FileSpec("pi", new FuncSpec(pigStorage)), - null) ; + null, true) ; return load ; } catch (IOException e) { throw new AssertionError("This cannot happen") ; Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=692253&r1=692252&r2=692253&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java Thu Sep 4 14:25:41 2008 @@ -94,7 +94,7 @@ FileSpec fileSpec = new FileSpec("pi", new FuncSpec(PigStorage.class.getName())) ; try { - load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ; + load = new LOLoad(plan, getKey(node.attributes), fileSpec, null, true) ; fillSchema(load, node.attributes) ; } catch (IOException ioe) {
