Author: pradeepkth
Date: Tue May 4 18:42:04 2010
New Revision: 940995
URL: http://svn.apache.org/viewvc?rev=940995&view=rev
Log:
"explain -script <script file>" executes grunt commands like run/dump/copy etc
- explain -script should not execute any grunt command and only explain the
query plans (pradeepkth)
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=940995&r1=940994&r2=940995&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue May 4 18:42:04 2010
@@ -53,6 +53,10 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1401: "explain -script <script file>" executes grunt commands like
+run/dump/copy etc - explain -script should not execute any grunt command and
+only explain the query plans (pradeepkth)
+
PIG-1303: Inconsistent instantiation of parametrized UDFs (jrussek and
dvryaboy)
740 : Incorrect line number is generated when a string with double quotes is
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=940995&r1=940994&r2=940995&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Tue May 4
18:42:04 2010
@@ -224,10 +224,14 @@ public class GruntParser extends PigScri
@Override
protected void processDescribe(String alias) throws IOException {
- if(alias==null) {
- alias = mPigServer.getPigContext().getLastAlias();
+ if(mExplain == null) { // process only if not in "explain" mode
+ if(alias==null) {
+ alias = mPigServer.getPigContext().getLastAlias();
+ }
+ mPigServer.dumpSchema(alias);
+ } else {
+ log.warn("'describe' statement is ignored while processing
explain");
}
- mPigServer.dumpSchema(alias);
}
@Override
@@ -317,7 +321,11 @@ public class GruntParser extends PigScri
@Override
protected void printAliases() throws IOException {
- mPigServer.printAliases();
+ if(mExplain == null) { // process only if not in "explain" mode
+ mPigServer.printAliases();
+ } else {
+ log.warn("'aliases' statement is ignored while processing
explain");
+ }
}
@Override
@@ -349,22 +357,26 @@ public class GruntParser extends PigScri
List<String> params, List<String> files)
throws IOException, ParseException {
- if (script == null) {
- executeBatch();
- return;
- }
-
- if (batch) {
- setBatchOn();
- mPigServer.setJobName(script);
- try {
- loadScript(script, true, mLoadOnly, params, files);
+ if(mExplain == null) { // process only if not in "explain" mode
+ if (script == null) {
executeBatch();
- } finally {
- discardBatch();
+ return;
+ }
+
+ if (batch) {
+ setBatchOn();
+ mPigServer.setJobName(script);
+ try {
+ loadScript(script, true, mLoadOnly, params, files);
+ executeBatch();
+ } finally {
+ discardBatch();
+ }
+ } else {
+ loadScript(script, false, mLoadOnly, params, files);
}
} else {
- loadScript(script, false, mLoadOnly, params, files);
+ log.warn("'run/exec' statement is ignored while processing
explain");
}
}
@@ -465,44 +477,49 @@ public class GruntParser extends PigScri
@Override
protected void processCat(String path) throws IOException
{
- executeBatch();
-
- try {
- byte buffer[] = new byte[65536];
- ElementDescriptor dfsPath = mDfs.asElement(path);
- int rc;
+ if(mExplain == null) { // process only if not in "explain" mode
- if (!dfsPath.exists())
- throw new IOException("Directory " + path + " does not
exist.");
-
- if (mDfs.isContainer(path)) {
- ContainerDescriptor dfsDir = (ContainerDescriptor) dfsPath;
- Iterator<ElementDescriptor> paths = dfsDir.iterator();
+ executeBatch();
+
+ try {
+ byte buffer[] = new byte[65536];
+ ElementDescriptor dfsPath = mDfs.asElement(path);
+ int rc;
- while (paths.hasNext()) {
- ElementDescriptor curElem = paths.next();
+ if (!dfsPath.exists())
+ throw new IOException("Directory " + path + " does not
exist.");
+
+ if (mDfs.isContainer(path)) {
+ ContainerDescriptor dfsDir = (ContainerDescriptor) dfsPath;
+ Iterator<ElementDescriptor> paths = dfsDir.iterator();
- if (mDfs.isContainer(curElem.toString())) {
- continue;
+ while (paths.hasNext()) {
+ ElementDescriptor curElem = paths.next();
+
+ if (mDfs.isContainer(curElem.toString())) {
+ continue;
+ }
+
+ InputStream is = curElem.open();
+ while ((rc = is.read(buffer)) > 0) {
+ System.out.write(buffer, 0, rc);
+ }
+ is.close();
}
-
- InputStream is = curElem.open();
+ }
+ else {
+ InputStream is = dfsPath.open();
while ((rc = is.read(buffer)) > 0) {
System.out.write(buffer, 0, rc);
}
- is.close();
+ is.close();
}
}
- else {
- InputStream is = dfsPath.open();
- while ((rc = is.read(buffer)) > 0) {
- System.out.write(buffer, 0, rc);
- }
- is.close();
+ catch (DataStorageException e) {
+ throw new IOException("Failed to Cat: " + path, e);
}
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to Cat: " + path, e);
+ } else {
+ log.warn("'cat' statement is ignored while processing explain");
}
}
@@ -510,49 +527,60 @@ public class GruntParser extends PigScri
protected void processCD(String path) throws IOException
{
ContainerDescriptor container;
-
- try {
- if (path == null) {
- container = mDfs.asContainer("/user/" +
System.getProperty("user.name"));
- mDfs.setActiveContainer(container);
- }
- else
- {
- container = mDfs.asContainer(path);
-
- if (!container.exists()) {
- throw new IOException("Directory " + path + " does not
exist.");
+ if(mExplain == null) { // process only if not in "explain" mode
+ try {
+ if (path == null) {
+ container = mDfs.asContainer("/user/" +
System.getProperty("user.name"));
+ mDfs.setActiveContainer(container);
}
-
- if (!mDfs.isContainer(path)) {
- throw new IOException(path + " is not a directory.");
+ else
+ {
+ container = mDfs.asContainer(path);
+
+ if (!container.exists()) {
+ throw new IOException("Directory " + path + " does not
exist.");
+ }
+
+ if (!mDfs.isContainer(path)) {
+ throw new IOException(path + " is not a directory.");
+ }
+
+ mDfs.setActiveContainer(container);
}
-
- mDfs.setActiveContainer(container);
}
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to change working directory to " +
- ((path == null) ? ("/user/" +
System.getProperty("user.name"))
- : (path)), e);
+ catch (DataStorageException e) {
+ throw new IOException("Failed to change working directory to "
+
+ ((path == null) ? ("/user/" +
System.getProperty("user.name"))
+ : (path)), e);
+ }
+ } else {
+ log.warn("'cd' statement is ignored while processing explain");
}
}
@Override
protected void processDump(String alias) throws IOException
{
- Iterator<Tuple> result = mPigServer.openIterator(alias);
- while (result.hasNext())
- {
- Tuple t = result.next();
- System.out.println(TupleFormat.format(t));
+ if(mExplain == null) { // process only if not in "explain" mode
+ Iterator<Tuple> result = mPigServer.openIterator(alias);
+ while (result.hasNext())
+ {
+ Tuple t = result.next();
+ System.out.println(TupleFormat.format(t));
+ }
+ } else {
+ log.warn("'dump' statement is ignored while processing explain");
}
}
@Override
protected void processIllustrate(String alias) throws IOException
{
- mPigServer.getExamples(alias);
+ if(mExplain == null) { // process only if not in "explain" mode
+ mPigServer.getExamples(alias);
+ } else {
+ log.warn("'illustrate' statement is ignored while processing
explain");
+ }
}
@Override
@@ -575,39 +603,43 @@ public class GruntParser extends PigScri
@Override
protected void processLS(String path) throws IOException
{
- try {
- ElementDescriptor pathDescriptor;
-
- if (path == null) {
- pathDescriptor = mDfs.getActiveContainer();
- }
- else {
- pathDescriptor = mDfs.asElement(path);
- }
-
- if (!pathDescriptor.exists()) {
- throw new IOException("File or directory " + path + " does not
exist.");
- }
-
- if (mDfs.isContainer(pathDescriptor.toString())) {
- ContainerDescriptor container = (ContainerDescriptor)
pathDescriptor;
- Iterator<ElementDescriptor> elems = container.iterator();
+ if(mExplain == null) { // process only if not in "explain" mode
+ try {
+ ElementDescriptor pathDescriptor;
+
+ if (path == null) {
+ pathDescriptor = mDfs.getActiveContainer();
+ }
+ else {
+ pathDescriptor = mDfs.asElement(path);
+ }
+
+ if (!pathDescriptor.exists()) {
+ throw new IOException("File or directory " + path + " does
not exist.");
+ }
- while (elems.hasNext()) {
- ElementDescriptor curElem = elems.next();
+ if (mDfs.isContainer(pathDescriptor.toString())) {
+ ContainerDescriptor container = (ContainerDescriptor)
pathDescriptor;
+ Iterator<ElementDescriptor> elems = container.iterator();
- if (mDfs.isContainer(curElem.toString())) {
- System.out.println(curElem.toString() + "\t<dir>");
- } else {
- printLengthAndReplication(curElem);
+ while (elems.hasNext()) {
+ ElementDescriptor curElem = elems.next();
+
+ if (mDfs.isContainer(curElem.toString())) {
+ System.out.println(curElem.toString() +
"\t<dir>");
+ } else {
+ printLengthAndReplication(curElem);
+ }
}
+ } else {
+ printLengthAndReplication(pathDescriptor);
}
- } else {
- printLengthAndReplication(pathDescriptor);
}
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to LS on " + path, e);
+ catch (DataStorageException e) {
+ throw new IOException("Failed to LS on " + path, e);
+ }
+ } else {
+ log.warn("'ls' statement is ignored while processing explain");
}
}
@@ -625,7 +657,11 @@ public class GruntParser extends PigScri
@Override
protected void processPWD() throws IOException
{
- System.out.println(mDfs.getActiveContainer().toString());
+ if(mExplain == null) { // process only if not in "explain" mode
+ System.out.println(mDfs.getActiveContainer().toString());
+ } else {
+ log.warn("'pwd' statement is ignored while processing explain");
+ }
}
@Override
@@ -650,76 +686,100 @@ public class GruntParser extends PigScri
@Override
protected void processMove(String src, String dst) throws IOException
{
- executeBatch();
+ if(mExplain == null) { // process only if not in "explain" mode
- try {
- ElementDescriptor srcPath = mDfs.asElement(src);
- ElementDescriptor dstPath = mDfs.asElement(dst);
-
- if (!srcPath.exists()) {
- throw new IOException("File or directory " + src + " does not
exist.");
+ executeBatch();
+
+ try {
+ ElementDescriptor srcPath = mDfs.asElement(src);
+ ElementDescriptor dstPath = mDfs.asElement(dst);
+
+ if (!srcPath.exists()) {
+ throw new IOException("File or directory " + src + " does
not exist.");
+ }
+
+ srcPath.rename(dstPath);
}
-
- srcPath.rename(dstPath);
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to move " + src + " to " + dst, e);
+ catch (DataStorageException e) {
+ throw new IOException("Failed to move " + src + " to " + dst,
e);
+ }
+ } else {
+ log.warn("'mv' statement is ignored while processing explain");
}
}
@Override
protected void processCopy(String src, String dst) throws IOException
{
- executeBatch();
+ if(mExplain == null) { // process only if not in "explain" mode
- try {
- ElementDescriptor srcPath = mDfs.asElement(src);
- ElementDescriptor dstPath = mDfs.asElement(dst);
-
- srcPath.copy(dstPath, mConf, false);
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to copy " + src + " to " + dst, e);
+ executeBatch();
+
+ try {
+ ElementDescriptor srcPath = mDfs.asElement(src);
+ ElementDescriptor dstPath = mDfs.asElement(dst);
+
+ srcPath.copy(dstPath, mConf, false);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to copy " + src + " to " + dst,
e);
+ }
+ } else {
+ log.warn("'cp' statement is ignored while processing explain");
}
}
@Override
protected void processCopyToLocal(String src, String dst) throws
IOException
{
- executeBatch();
-
- try {
- ElementDescriptor srcPath = mDfs.asElement(src);
- ElementDescriptor dstPath = mLfs.asElement(dst);
+ if(mExplain == null) { // process only if not in "explain" mode
- srcPath.copy(dstPath, false);
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to copy " + src + "to (locally) " +
dst, e);
+ executeBatch();
+
+ try {
+ ElementDescriptor srcPath = mDfs.asElement(src);
+ ElementDescriptor dstPath = mLfs.asElement(dst);
+
+ srcPath.copy(dstPath, false);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to copy " + src + "to (locally)
" + dst, e);
+ }
+ } else {
+ log.warn("'copyToLocal' statement is ignored while processing
explain");
}
}
@Override
protected void processCopyFromLocal(String src, String dst) throws
IOException
{
- executeBatch();
-
- try {
- ElementDescriptor srcPath = mLfs.asElement(src);
- ElementDescriptor dstPath = mDfs.asElement(dst);
+ if(mExplain == null) { // process only if not in "explain" mode
- srcPath.copy(dstPath, false);
- }
- catch (DataStorageException e) {
- throw new IOException("Failed to copy (loally) " + src + "to " +
dst, e);
+ executeBatch();
+
+ try {
+ ElementDescriptor srcPath = mLfs.asElement(src);
+ ElementDescriptor dstPath = mDfs.asElement(dst);
+
+ srcPath.copy(dstPath, false);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to copy (loally) " + src + "to "
+ dst, e);
+ }
+ } else {
+ log.warn("'copyFromLocal' statement is ignored while processing
explain");
}
}
@Override
protected void processMkdir(String dir) throws IOException
{
- ContainerDescriptor dirDescriptor = mDfs.asContainer(dir);
- dirDescriptor.create();
+ if(mExplain == null) { // process only if not in "explain" mode
+ ContainerDescriptor dirDescriptor = mDfs.asContainer(dir);
+ dirDescriptor.create();
+ } else {
+ log.warn("'mkdir' statement is ignored while processing explain");
+ }
}
@Override
@@ -741,27 +801,38 @@ public class GruntParser extends PigScri
@Override
protected void processRemove(String path, String options ) throws
IOException
{
- ElementDescriptor dfsPath = mDfs.asElement(path);
+ if(mExplain == null) { // process only if not in "explain" mode
- executeBatch();
+ ElementDescriptor dfsPath = mDfs.asElement(path);
+ executeBatch();
- if (!dfsPath.exists()) {
- if (options == null || !options.equalsIgnoreCase("force")) {
- throw new IOException("File or directory " + path + " does not
exist.");
+ if (!dfsPath.exists()) {
+ if (options == null || !options.equalsIgnoreCase("force")) {
+ throw new IOException("File or directory " + path + " does
not exist.");
+ }
}
- }
- else {
-
- dfsPath.delete();
+ else {
+
+ dfsPath.delete();
+ }
+ } else {
+ log.warn("'rm/rmf' statement is ignored while processing explain");
}
}
@Override
protected void processFsCommand(String[] cmdTokens) throws IOException{
- try {
- shell.run(cmdTokens);
- } catch (Exception e) {
- throw new IOException(e);
+ if(mExplain == null) { // process only if not in "explain" mode
+
+ executeBatch();
+
+ try {
+ shell.run(cmdTokens);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ } else {
+ log.warn("'fs' statement is ignored while processing explain");
}
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=940995&r1=940994&r2=940995&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Tue May 4
18:42:04 2010
@@ -21,6 +21,9 @@ import org.junit.Test;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.PatternLayout;
import org.apache.pig.ExecType;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
@@ -32,6 +35,8 @@ import org.apache.pig.tools.pigscript.pa
import org.apache.pig.impl.util.LogUtils;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileReader;
import java.io.InputStreamReader;
import java.io.BufferedReader;
import java.io.StringReader;
@@ -65,6 +70,7 @@ public class TestGrunt extends TestCase
grunt.exec();
}
*/
+
@Test
public void testDefine() throws Throwable {
PigServer server = new PigServer(ExecType.MAPREDUCE,
cluster.getProperties());
@@ -450,6 +456,54 @@ public class TestGrunt extends TestCase
grunt.exec();
}
+ /**
+ * verify that grunt commands are ignored in explain -script mode
+ */
+ @Test
+ public void testExplainScript2() throws Throwable {
+
+ PigServer server = new PigServer(ExecType.MAPREDUCE,
cluster.getProperties());
+ PigContext context = server.getPigContext();
+
+ String strCmd = "explain -script "
+ + basedir + "/explainScript.pig;";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ String logMessagesFile = "TestGrunt-testExplainScript2-stderr.txt";
+ // add a file based appender to the root logger so we can parse the
+ // messages logged by grunt and verify that grunt commands are ignored
+ // in explain -script mode
+ Appender fileAppender = new FileAppender(new PatternLayout(),
logMessagesFile);
+
+ try {
+
org.apache.log4j.LogManager.getRootLogger().addAppender(fileAppender);
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ BufferedReader in = new BufferedReader(new
FileReader(logMessagesFile));
+ String gruntLoggingContents = "";
+ //read file into a string
+ String line;
+ while ( (line = in.readLine()) != null) {
+ gruntLoggingContents += line + "\n";
+ }
+ in.close();
+ String[] cmds = new String[] { "'rm/rmf'", "'cp'", "'cat'",
"'cd'", "'pwd'",
+ "'copyFromLocal'", "'copyToLocal'", "'describe'", "'ls'",
+ "'mkdir'", "'illustrate'", "'run/exec'", "'fs'",
"'aliases'",
+ "'mv'", "'dump'" };
+ for (String c : cmds) {
+ String expected = c + " statement is ignored while processing
explain";
+ assertTrue("Checking if " + gruntLoggingContents + " contains
" +
+ expected, gruntLoggingContents.contains(expected));
+ }
+ } finally {
+
org.apache.log4j.LogManager.getRootLogger().removeAppender(fileAppender);
+ new File(logMessagesFile).delete();
+ }
+ }
+
@Test
public void testExplainBrief() throws Throwable {
PigServer server = new PigServer(ExecType.MAPREDUCE,
cluster.getProperties());
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig?rev=940995&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig Tue May 4
18:42:04 2010
@@ -0,0 +1 @@
+rmf input-copy.txt; cat 'foo'; a = load '1.txt' ; aliases;illustrate a;
copyFromLocal foo bar; copyToLocal foo bar; describe a; mkdir foo; run bar.pig;
exec bar.pig; cp foo bar; explain a;cd 'bar'; pwd; ls ; fs -ls ; fs -rmr foo;
mv foo bar; dump a;store a into 'input-copy.txt' ; a = load '2.txt' as
(b);explain a; rm foo; store a into 'bar';
