Author: daijy
Date: Tue Jan 20 22:11:17 2015
New Revision: 1653386
URL: http://svn.apache.org/r1653386
Log:
PIG-4381: PIG grunt shell DEFINE commands fails when it spans multiple lines
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1653386&r1=1653385&r2=1653386&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 20 22:11:17 2015
@@ -44,6 +44,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4381: PIG grunt shell DEFINE commands fails when it spans multiple lines
(daijy)
+
PIG-4384: TezLauncher thread should be deamon thread (zjffdu via daijy)
PIG-4376: NullPointerException accessing a field of an invalid bag from a
nested foreach
Modified: pig/trunk/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1653386&r1=1653385&r2=1653386&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Tue Jan 20 22:11:17 2015
@@ -546,7 +546,7 @@ public class Main {
// Interactive
mode = ExecMode.SHELL;
//Reader is created by first loading
"pig.load.default.statements" or .pigbootup file if available
- ConsoleReader reader = new
ConsoleReaderWithParamSub(Utils.getCompositeStream(System.in, properties), new
OutputStreamWriter(System.out), pigContext);
+ ConsoleReader reader = new
ConsoleReader(Utils.getCompositeStream(System.in, properties), new
OutputStreamWriter(System.out));
reader.setDefaultPrompt("grunt> ");
final String HISTORYFILE = ".pig_history";
String historyFile = System.getProperty("user.home") +
File.separator + HISTORYFILE;
@@ -1073,23 +1073,4 @@ public class Main {
: ReturnCode.SUCCESS;
}
- static class ConsoleReaderWithParamSub extends ConsoleReader {
- PigContext pc;
- ConsoleReaderWithParamSub(InputStream in, Writer out, PigContext
pigContext) throws IOException {
- super(in, out);
- pc = pigContext;
- }
-
- @Override
- public String readLine() throws IOException {
- String line = super.readLine();
- if (null == line) {
- return line;
- }
- String paramSubLine = pc.doParamSubstitution(new
BufferedReader(new StringReader(line)));
- return paramSubLine;
- }
-
- }
-
}
Modified: pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1653386&r1=1653385&r2=1653386&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java Tue Jan 20
22:11:17 2015
@@ -379,4 +379,11 @@ public class DryRunGruntParser extends P
protected void printClear() {
}
+ @Override
+ protected void processDefault(String key, String value) throws IOException
{
+ }
+
+ @Override
+ protected void processDeclare(String key, String value) throws IOException
{
+ }
}
Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1653386&r1=1653385&r2=1653386&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Tue Jan 20
22:11:17 2015
@@ -451,12 +451,16 @@ public class GruntParser extends PigScri
@Override
protected void processRegister(String jar) throws IOException {
filter.validate(PigCommandFilter.Command.REGISTER);
+ jar = parameterSubstitutionInGrunt(jar);
mPigServer.registerJar(jar);
}
@Override
protected void processRegister(String path, String scriptingLang, String
namespace) throws IOException, ParseException {
filter.validate(PigCommandFilter.Command.REGISTER);
+ path = parameterSubstitutionInGrunt(path);
+ scriptingLang = parameterSubstitutionInGrunt(scriptingLang);
+ namespace = parameterSubstitutionInGrunt(namespace);
if(path.endsWith(".jar")) {
if(scriptingLang != null || namespace != null) {
throw new ParseException("Cannot register a jar with a
scripting language or namespace");
@@ -568,6 +572,8 @@ public class GruntParser extends PigScri
@Override
protected void processSet(String key, String value) throws IOException,
ParseException {
filter.validate(PigCommandFilter.Command.SET);
+ key = parameterSubstitutionInGrunt(key);
+ value = parameterSubstitutionInGrunt(value);
if (key.equals("debug"))
{
if (value.equals("on"))
@@ -636,6 +642,7 @@ public class GruntParser extends PigScri
@Override
protected void processCat(String path) throws IOException {
filter.validate(PigCommandFilter.Command.CAT);
+ path = parameterSubstitutionInGrunt(path);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -685,6 +692,7 @@ public class GruntParser extends PigScri
@Override
protected void processCD(String path) throws IOException {
filter.validate(PigCommandFilter.Command.CD);
+ path = parameterSubstitutionInGrunt(path);
ContainerDescriptor container;
if(mExplain == null) { // process only if not in "explain" mode
@@ -820,6 +828,7 @@ public class GruntParser extends PigScri
@Override
protected void processLS(String path) throws IOException {
filter.validate(PigCommandFilter.Command.LS);
+ path = parameterSubstitutionInGrunt(path);
if (mExplain == null) { // process only if not in "explain" mode
@@ -945,6 +954,8 @@ public class GruntParser extends PigScri
protected void processMove(String src, String dst) throws IOException
{
filter.validate(PigCommandFilter.Command.MV);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -971,6 +982,8 @@ public class GruntParser extends PigScri
protected void processCopy(String src, String dst) throws IOException
{
filter.validate(PigCommandFilter.Command.CP);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -993,6 +1006,8 @@ public class GruntParser extends PigScri
protected void processCopyToLocal(String src, String dst) throws
IOException
{
filter.validate(PigCommandFilter.Command.COPYTOLOCAL);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1015,6 +1030,8 @@ public class GruntParser extends PigScri
protected void processCopyFromLocal(String src, String dst) throws
IOException
{
filter.validate(PigCommandFilter.Command.COPYFROMLOCAL);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1037,6 +1054,7 @@ public class GruntParser extends PigScri
protected void processMkdir(String dir) throws IOException
{
filter.validate(PigCommandFilter.Command.MKDIR);
+ dir = parameterSubstitutionInGrunt(dir);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1052,6 +1070,7 @@ public class GruntParser extends PigScri
protected void processPig(String cmd) throws IOException
{
int start = 1;
+ cmd = parameterSubstitutionInGrunt(cmd);
if (!mInteractive) {
start = getLineNumber();
}
@@ -1068,6 +1087,7 @@ public class GruntParser extends PigScri
protected void processRemove(String path, String options) throws
IOException {
filter.validate(PigCommandFilter.Command.RM);
filter.validate(PigCommandFilter.Command.RMF);
+ path = parameterSubstitutionInGrunt(path);
int MAX_MS_TO_WAIT_FOR_FILE_DELETION = 10 * 60 * 1000;
int MS_TO_SLEEP_WHILE_WAITING_FOR_FILE_DELETION = 250;
@@ -1112,6 +1132,9 @@ public class GruntParser extends PigScri
@Override
protected void processFsCommand(String[] cmdTokens) throws IOException {
filter.validate(PigCommandFilter.Command.FS);
+ for (int i = 0 ; i < cmdTokens.length ; i++) {
+ cmdTokens[i] = parameterSubstitutionInGrunt(cmdTokens[i]);
+ }
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1138,6 +1161,9 @@ public class GruntParser extends PigScri
@Override
protected void processShCommand(String[] cmdTokens) throws IOException {
filter.validate(PigCommandFilter.Command.SH);
+ for (int i = 0 ; i < cmdTokens.length ; i++) {
+ cmdTokens[i] = parameterSubstitutionInGrunt(cmdTokens[i]);
+ }
if(mExplain == null) { // process only if not in "explain" mode
try {
executeBatch();
@@ -1241,6 +1267,7 @@ public class GruntParser extends PigScri
@Override
protected void processSQLCommand(String cmd) throws IOException{
+ cmd = parameterSubstitutionInGrunt(cmd);
if(mExplain == null) { // process only if not in "explain" mode
if
(!mPigServer.getPigContext().getProperties().get("pig.sql.type").equals("hcat"))
{
throw new IOException("sql command only support hcat
currently");
@@ -1259,6 +1286,22 @@ public class GruntParser extends PigScri
}
}
+ @Override
+ protected void processDefault(String key, String value) throws IOException
{
+ parameterSubstitutionInGrunt("%default " + key + " " + value);
+ }
+
+ @Override
+ protected void processDeclare(String key, String value) throws IOException
{
+ parameterSubstitutionInGrunt("%declare " + key + " " + value);
+ }
+
+ private String parameterSubstitutionInGrunt(String input) throws
IOException {
+ if (mInteractive && input != null) {
+ return mPigServer.getPigContext().doParamSubstitution(new
BufferedReader(new StringReader(input))).trim();
+ }
+ return input;
+ }
/**
* StreamPrinter.
*
Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1653386&r1=1653385&r2=1653386&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
(original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Tue
Jan 20 22:11:17 2015
@@ -120,6 +120,10 @@ public abstract class PigScriptParser
abstract protected void processScript(String script, boolean batch,
List<String> params, List<String> files) throws IOException, ParseException;
+ abstract protected void processDefault(String key, String value) throws
IOException;
+
+ abstract protected void processDeclare(String key, String value) throws
IOException;
+
static String unquote(String s)
{
if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
@@ -186,6 +190,8 @@ TOKEN: {<XML: "-xml">}
TOKEN: {<OUT: "-out">}
TOKEN: {<BRIEF: "-brief">}
TOKEN: {<N: "-n">}
+TOKEN: {<PIGDEFAULT: "%default" >}
+TOKEN: {<DECLARE: "%declare" >}
// internal use commands
TOKEN: {<SCRIPT_DONE: "scriptDone">}
@@ -420,6 +426,7 @@ TOKEN : { <QUOTEDSTRING : "'"
)
)*
"'"> }
+TOKEN: {<SHELLCMD: "`" (~["`"])* "`" >}
void parse() throws IOException:
{
Token t1, t2, t3;
@@ -598,6 +605,16 @@ void parse() throws IOException:
{processSet();}
)
|
+ <PIGDEFAULT>
+ t1 = GetKey()
+ t2 = GetDefaultValue()
+ {processDefault(t1.image, t2.image);}
+ |
+ <DECLARE>
+ t1 = GetKey()
+ t2 = GetDefaultValue()
+ {processDeclare(t1.image, t2.image);}
+ |
<EOF>
{quit();}
|
@@ -796,6 +813,22 @@ Token GetValue() :
)
{return t;}
+}
+
+Token GetDefaultValue() :
+{
+ Token t;
+}
+{
+ (
+ t = GetPath()
+ |
+ t = <QUOTEDSTRING>
+ |
+ t = <SHELLCMD>
+ )
+
+ {return t;}
}
Token GetReserved () :
Modified: pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1653386&r1=1653385&r2=1653386&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Tue Jan 20
22:11:17 2015
@@ -1356,12 +1356,39 @@ public class TestParamSubPreproc {
File outputFile = File.createTempFile("tmp", "");
outputFile.delete();
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
- String command = "a = load '" + Util.generateURI(inputFile.toString(),
pc) + "' as ($param1:chararray, $param2:int);\n"
- + "store a into '" + Util.generateURI(outputFile.toString(),
pc) + "';\n"
+ String command = "%default agelimit `echo 15`\n"
+ + "rmf $outputFile;\n"
+ + "a = load '" + Util.generateURI(inputFile.toString(), pc) +
"' as ($param1:chararray, $param2:int);\n"
+ + "b = filter a by age > $agelimit;"
+ + "store b into '$outputFile';\n"
+ "quit\n";
System.setProperty("jline.WindowsTerminal.directConsole", "false");
System.setIn(new ByteArrayInputStream(command.getBytes()));
- org.apache.pig.PigRunner.run(new String[] {"-x", "local", "-p",
"param1=name", "-p", "param2=age"}, null);
+ org.apache.pig.PigRunner.run(new String[] {"-x", "local", "-p",
"param1=name", "-p", "param2=age", "-p", "outputFile=" +
Util.generateURI(outputFile.toString(), pc)}, null);
+ File[] partFiles = outputFile.listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith("part");
+ }
+ });
+ String resultContent = Util.readFile(partFiles[0]);
+ assertEquals(resultContent, "jenny\t20\n");
+ }
+
+ @Test
+ public void testGruntMultilineDefine() throws Exception{
+ log.info("Starting test testGruntMultilineDefine()");
+ File inputFile = Util.createFile(new
String[]{"daniel\t10","jenny\t20"});
+ File outputFile = File.createTempFile("tmp", "");
+ outputFile.delete();
+ PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
+ String command = "DEFINE process(input_file) returns data {\n" +
+ "$data = load '$input_file' using PigStorage(',');};\n" +
+ "b = process('" + Util.generateURI(inputFile.toString(), pc)
+ "');\n" +
+ "store b into '" + Util.generateURI(outputFile.toString(), pc)
+ "';" +
+ "quit\n";
+ System.setProperty("jline.WindowsTerminal.directConsole", "false");
+ System.setIn(new ByteArrayInputStream(command.getBytes()));
+ org.apache.pig.PigRunner.run(new String[] {"-x", "local"}, null);
File[] partFiles = outputFile.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.startsWith("part");
