Author: knoguchi
Date: Mon Mar 21 21:45:23 2016
New Revision: 1736101
URL: http://svn.apache.org/viewvc?rev=1736101&view=rev
Log:
PIG-4841: Inline-op with schema declaration fails with syntax error (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
pig/trunk/test/org/apache/pig/test/TestGrunt.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1736101&r1=1736100&r2=1736101&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Mar 21 21:45:23 2016
@@ -99,6 +99,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4841: Inline-op with schema declaration fails with syntax error (knoguchi)
+
PIG-4832: Fix TestPrumeColumn NPE failure (kellyzly via daijy)
PIG-4833 TestBuiltin.testURIWithCurlyBrace in TEZ failing after PIG-4819
(knoguchi)
Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1736101&r1=1736100&r2=1736101&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
(original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Mon
Mar 21 21:45:23 2016
@@ -355,7 +355,20 @@ TOKEN_MGR_DECLS : {
<SCHEMA_DEFINITION> MORE :
{
<"("> {tupleSchemaLevel++;}
-| <")"> {tupleSchemaLevel--; if ((tupleSchemaLevel == 0) && (bagSchemaLevel
== 0)) SwitchTo(prevState); }
+| <")">
+ {
+ if ((tupleSchemaLevel == 0) && (bagSchemaLevel == 0)) {
+ // This means parenthesis is not from this schema_def.
+ // Putting back ")" although others do not check parenthesis at this
time.
+ // This is a bandaid workaround for the issue with inline-op which
+ // also uses parenthesis.
+ // Real fix would be to move out of using this javacc parser.
(PIG-2597)
+ input_stream.backup(1);
+ image.deleteCharAt(image.length()-1);
+ SwitchTo(prevState);
+ }
+ tupleSchemaLevel--; if ((tupleSchemaLevel == 0) && (bagSchemaLevel ==
0)) SwitchTo(prevState);
+ }
| <"{"> {bagSchemaLevel++;}
| <"}"> {bagSchemaLevel--; if ((tupleSchemaLevel == 0) && (bagSchemaLevel ==
0)) SwitchTo(prevState); }
| <("," | ";" )>
Modified: pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=1736101&r1=1736100&r2=1736101&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestGrunt.java Mon Mar 21 21:45:23 2016
@@ -1363,6 +1363,39 @@ public class TestGrunt {
validate(query, false, msgs.toArray(new String[0]));
}
+ @Test
+ public void testWithInlineOp() throws Throwable {
+ // specifying schema inside inline-op makes PigScriptParser.jj to read
+ // to the end of file
+ String query = "a = load 'i1' as (f1:chararray);" +
+ "b = foreach (foreach a generate f1 as b1) generate b1; " +
+ "dump b; ";
+
+ ArrayList<String> msgs = new ArrayList<String>(); //
+ validate(query, true, msgs.toArray(new String[0]));
+ }
+
+ /*
+ * Following test currently fails. Insead of making further changes to
+ * PigScriptParser.jj, leaving it till we move out of javacc in PIG-2597
+
+ @Test
+ public void testWithInlineOpWithNestedForeach() throws Throwable {
+ // This one currently fails because "{}" is treated as
+ // IN_BLOCK in PigScriptParser.jj which jumps to PIG_END and ignore
+ // ") generate *; " part of the code.
+ // In order to support this test, we need to add parenthesis matching
+ // everywhere in PigScriptParser.jj (or stop using it)
+ //
+ String query = "a = load 'i1' as (f1:chararray);" +
+ "b = group a ALL; " +
+ "c = foreach ( foreach b {b1 = limit a 3; generate 1, b1;} )
generate *; " +
+ "dump c;";
+ ArrayList<String> msgs = new ArrayList<String>(); //
+ validate(query, true, msgs.toArray(new String[0]));
+ }
+ */
+
private void validate(String query, boolean syntaxOk,
