Author: rohini
Date: Mon Dec 4 20:28:18 2017
New Revision: 1817125
URL: http://svn.apache.org/viewvc?rev=1817125&view=rev
Log:
PIG-5315: pig.script is not set for scripts run via PigServer (satishsaley via
rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java
pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Dec 4 20:28:18 2017
@@ -60,6 +60,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5315: pig.script is not set for scripts run via PigServer (satishsaley via
rohini)
+
PIG-5310: MergeJoin throwing NullPointer Exception (satishsaley via rohini)
PIG-5314: Abort method is not implemented in PigProcessor (satishsaley via
rohini)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Mon Dec 4 20:28:18 2017
@@ -43,6 +43,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -775,7 +776,10 @@ public class PigServer {
*/
public void registerScript(InputStream in, Map<String,String>
params,List<String> paramsFiles) throws IOException {
try {
- String substituted = pigContext.doParamSubstitution(in,
paramMapToList(params), paramsFiles);
+ String script = IOUtils.toString(in);
+ ScriptState.get().setScript(script);
+ String substituted = pigContext.doParamSubstitution(new
BufferedReader(new StringReader(script)),
+ paramMapToList(params), paramsFiles);
GruntParser grunt = new GruntParser(new StringReader(substituted),
this);
grunt.setInteractive(false);
grunt.parseStopOnError(true);
@@ -855,6 +859,7 @@ public class PigServer {
FileInputStream fis = null;
try{
fis = new FileInputStream(fileName);
+ ScriptState.get().setFileName(fileName);
registerScript(fis, params, paramsFiles);
}catch (FileNotFoundException e){
log.error(e.getLocalizedMessage());
Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Mon Dec 4
20:28:18 2017
@@ -73,6 +73,7 @@ import org.apache.pig.tools.pigscript.pa
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
+import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.pig.validator.BlackAndWhitelistFilter;
import org.apache.pig.validator.PigCommandFilter;
import org.fusesource.jansi.Ansi;
@@ -494,9 +495,12 @@ public class GruntParser extends PigScri
setBatchOn();
mPigServer.setJobName(script);
try {
- loadScript(script, true, false, mLoadOnly, params, files);
+ FetchFileRet scriptFile = FileLocalizer.fetchFile(mConf,
script);
+ ScriptState.get().beginNestedScript(scriptFile.file);
+ loadScript(scriptFile, script, true, false, mLoadOnly,
params, files);
executeBatch();
} finally {
+ ScriptState.get().endNestedScript();
discardBatch();
}
} else {
@@ -508,6 +512,11 @@ public class GruntParser extends PigScri
}
private void loadScript(String script, boolean batch, boolean loadOnly,
boolean illustrate,
+ List<String> params, List<String> files) throws IOException,
ParseException {
+ loadScript(FileLocalizer.fetchFile(mConf, script), script, batch,
loadOnly, illustrate, params, files);
+ }
+
+ private void loadScript(FetchFileRet fetchFile, String script, boolean
batch, boolean loadOnly, boolean illustrate,
List<String> params, List<String> files)
throws IOException, ParseException {
@@ -524,7 +533,6 @@ public class GruntParser extends PigScri
pc.setParamFiles(files);
try {
- FetchFileRet fetchFile = FileLocalizer.fetchFile(mConf, script);
String cmds = runPreprocessor(fetchFile.file.getAbsolutePath(),
params, files);
if (mInteractive && !batch) { // Write prompt and echo commands
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Mon Dec 4
20:28:18 2017
@@ -26,6 +26,7 @@ import java.util.BitSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Stack;
import java.util.UUID;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
@@ -182,6 +183,8 @@ public abstract class ScriptState {
protected List<PigProgressNotificationListener> listeners =
Lists.newArrayList();
+ private Stack<ScriptInfo> scripts = new Stack<>();
+
protected ScriptState(String id) {
this.id = id;
this.serializedScript = "";
@@ -439,6 +442,31 @@ public abstract class ScriptState {
return featureLongToString(scriptFeatures);
}
+ /**
+ * Stores information about the current script and pushes it onto a stack.
+ *
+ * @param scriptFile
+ * @throws IOException
+ */
+ public void beginNestedScript(File scriptFile) throws IOException {
+ ScriptInfo scriptInfo = new ScriptInfo();
+ scriptInfo.fileName = this.fileName;
+ scriptInfo.serializedScript = this.serializedScript;
+ scriptInfo.truncatedScript = this.truncatedScript;
+ scripts.push(scriptInfo);
+ this.setScript(scriptFile);
+ this.setFileName(scriptFile.getName());
+ }
+
+ public void endNestedScript() {
+ if (!scripts.isEmpty()) {
+ ScriptInfo scriptInfo = scripts.pop();
+ // Change the current script information
+ this.fileName = scriptInfo.fileName;
+ this.serializedScript = scriptInfo.serializedScript;
+ this.truncatedScript = scriptInfo.truncatedScript;
+ }
+ }
static class LogicalPlanFeatureVisitor extends
LogicalRelationalNodesVisitor {
private BitSet feature;
@@ -712,4 +740,10 @@ public abstract class ScriptState {
}
}
}
+
+ private static class ScriptInfo {
+ String serializedScript;
+ String truncatedScript;
+ String fileName;
+ }
}
Modified: pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java (original)
+++ pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java Mon Dec 4
20:28:18 2017
@@ -20,9 +20,11 @@ import java.io.InputStreamReader;
import java.util.Map;
import java.util.Properties;
+import org.apache.commons.io.IOUtils;
import org.apache.pig.ExecType;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.tools.pigstats.ScriptState;
/**
* Slightly modified PigServer that accepts a list of Pig aliases to override.
@@ -49,6 +51,8 @@ public class PigServer extends org.apach
public void registerScript(String fileName, Map<String, String>
aliasOverride)
throws IOException {
try {
+ ScriptState.get().setFileName(fileName);
+ ScriptState.get().setScript(IOUtils.toString(new
FileInputStream(fileName)));
InputStream compositeStream = Utils.getCompositeStream(new
FileInputStream(fileName), pigContext.getProperties());
GruntParser grunt = new GruntParser(new
InputStreamReader(compositeStream), this, aliasOverride);
grunt.setInteractive(false);
Modified: pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java Mon Dec 4
20:28:18 2017
@@ -20,6 +20,7 @@ package org.apache.pig.test;
import static org.apache.pig.builtin.mock.Storage.resetData;
import static org.apache.pig.builtin.mock.Storage.tuple;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -54,6 +55,7 @@ import org.apache.pig.scripting.ScriptEn
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.grunt.GruntParser;
import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.ScriptState;
import org.junit.Before;
import org.junit.Test;
@@ -167,6 +169,7 @@ public class TestPigServerLocal {
params.put("input", "test/org/apache/pig/test/data/passwd");
String script="a = load '$input' using PigStorage(':');";
pig.registerScript(new
ByteArrayInputStream(script.getBytes("UTF-8")),params);
+ assertEquals("ScriptState contains different script", script,
ScriptState.get().getScript());
Iterator<Tuple> iter=pig.openIterator("a");
int index=0;
List<Tuple>
expectedTuples=Util.readFile2TupleList("test/org/apache/pig/test/data/passwd",
":");
