Author: rding Date: Wed Mar 10 18:56:59 2010 New Revision: 921511 URL: http://svn.apache.org/viewvc?rev=921511&view=rev Log: PIG-1260: Param Subsitution results in parser error if there is no EOL after last line in script
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=921511&r1=921510&r2=921511&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Wed Mar 10 18:56:59 2010 @@ -145,6 +145,9 @@ OPTIMIZATIONS BUG FIXES +PIG-1260: Param Subsitution results in parser error if there is no EOL after +last line in script (rding) + PIG-1238: Dump does not respect the schema (rding) PIG-1261: PigStorageSchema broke after changes to ResourceSchema (dvryaboy via Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java?rev=921511&r1=921510&r2=921511&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java Wed Mar 10 18:56:59 2010 @@ -81,6 +81,19 @@ public class ParameterSubstitutionPrepro } } + // In case there is no EOL before EOF, add EOL for each line + String line = null; + StringBuilder blder = new StringBuilder(); + try { + while ((line = pigInput.readLine()) != null) { + blder.append(line).append("\n"); + } + } catch (IOException e) { + throw new ParseException(e.getMessage()); + } + + pigInput = new BufferedReader(new StringReader(blder.toString())); + // perform the substitution parsePigFile(pigInput , pigOutput); } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=921511&r1=921510&r2=921511&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Wed Mar 10 18:56:59 2010 @@ -27,12 +27,15 @@ import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.impl.PigContext; import org.apache.pig.tools.grunt.Grunt; +import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor; import org.apache.pig.tools.pigscript.parser.ParseException; import org.apache.pig.impl.util.LogUtils; import java.io.ByteArrayInputStream; import java.io.InputStreamReader; import java.io.BufferedReader; +import java.io.StringReader; +import java.io.StringWriter; public class TestGrunt extends TestCase { MiniCluster cluster = MiniCluster.buildCluster(); @@ -836,4 +839,23 @@ public class TestGrunt extends TestCase grunt.exec(); assertTrue(context.extraJars.contains(ClassLoader.getSystemResource("pig-withouthadoop.jar"))); } + + public void testScriptMissingLastNewLine() throws Throwable { + PigServer server = new PigServer(ExecType.LOCAL); + PigContext context = server.getPigContext(); + + String strCmd = "A = load 'bar';\nB = foreach A generate $0;"; + + ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50); + BufferedReader pin = new BufferedReader(new StringReader(strCmd)); + StringWriter writer = new StringWriter(); + psp.genSubstitutedFile(pin, writer, null, null); + pin = new BufferedReader(new StringReader(writer.toString())); + + Grunt grunt = new Grunt(pin, context); + int results[] = grunt.exec(); + for (int i=0; i<results.length; i++) { + assertTrue(results[i] == 0); + } + } }