Author: thejas
Date: Thu Sep 2 23:26:46 2010
New Revision: 992156
URL: http://svn.apache.org/viewvc?rev=992156&view=rev
Log:
PIG-1550: better error handling in casting relations to scalars
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/PigServer.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
hadoop/pig/trunk/test/org/apache/pig/test/TestScalarAliases.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=992156&r1=992155&r2=992156&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Sep 2 23:26:46 2010
@@ -193,6 +193,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1550: better error handling in casting relations to scalars (thejas)
+
PIG-1572: change default datatype when relations are used as scalar to
bytearray (thejas)
PIG-1583: piggybank unit test TestLookupInFiles is broken (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=992156&r1=992155&r2=992156&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Thu Sep 2 23:26:46 2010
@@ -733,14 +733,22 @@ public class PigServer {
} else if (job.getStatus() == JOB_STATUS.FAILED
&& job.getException() != null) {
// throw the backend exception in the failed case
- throw job.getException();
+ Exception e = job.getException();
+ int errCode = 1066;
+ String msg = "Unable to open iterator for alias " + id +
+ ". Backend error : " + e.getMessage();
+ throw new FrontendException(msg, errCode, PigException.INPUT,
e);
} else {
throw new IOException("Job terminated with anomalous status "
+ job.getStatus().toString());
}
- } catch (Exception e) {
+ }
+ catch(FrontendException e){
+ throw e;
+ }
+ catch (Exception e) {
int errCode = 1066;
- String msg = "Unable to open iterator for alias " + id;
+ String msg = "Unable to open iterator for alias " + id ;
throw new FrontendException(msg, errCode, PigException.INPUT, e);
}
}
@@ -808,8 +816,24 @@ public class PigServer {
throw new IOException("Couldn't retrieve job.");
}
OutputStats output = stats.getOutputStats().get(0);
- return new HJob(JOB_STATUS.COMPLETED, pigContext, output
- .getPOStore(), output.getAlias(), stats);
+
+ if(stats.isSuccessful()){
+ return new HJob(JOB_STATUS.COMPLETED, pigContext, output
+ .getPOStore(), output.getAlias(), stats);
+ }else{
+ HJob job = new HJob(JOB_STATUS.FAILED, pigContext,
+ output.getPOStore(), output.getAlias(), stats);
+
+ //check for exception
+ Exception ex = null;
+ for(JobStats js : stats.getJobGraph()){
+ if(js.getException() != null)
+ ex = js.getException();
+ }
+ job.setException(ex);
+ return job;
+ }
+
}
private PigStats storeEx(
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java?rev=992156&r1=992155&r2=992156&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
Thu Sep 2 23:26:46 2010
@@ -179,6 +179,7 @@ public abstract class Launcher {
String msgs[] = reports[i].getDiagnostics();
ArrayList<Exception> exceptions = new ArrayList<Exception>();
Set<String> errorMessageSet = new HashSet<String>();
+ String exceptionCreateFailMsg = null;
for (int j = 0; j < msgs.length; j++) {
if(!errorMessageSet.contains(msgs[j])) {
errorMessageSet.add(msgs[j]);
@@ -192,10 +193,9 @@ public abstract class Launcher {
Exception e = getExceptionFromString(msgs[j]);
exceptions.add(e);
} catch (Exception e1) {
+ // keep track of the exception we were unable to
re-create
String firstLine =
getFirstLineFromMessage(msgs[j]);
- int errCode = 2997;
- String msg = "Unable to recreate exception from
backed error: " + firstLine;
- throw new ExecException(msg, errCode,
PigException.BUG);
+ exceptionCreateFailMsg = firstLine;
}
} else {
log.debug("Error message from task (" + type + ") " +
@@ -203,6 +203,14 @@ public abstract class Launcher {
}
}
}
+
+ //if there are no valid exception that could be created, report
+ if(exceptions.size() == 0){
+ int errCode = 2997;
+ String msg = "Unable to recreate exception from backed error:
"
+ + exceptionCreateFailMsg;
+ throw new ExecException(msg, errCode, PigException.BUG);
+ }
//if its a failed job then check if there is more than one
exception
//more than one exception implies possibly different kinds of
failures
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=992156&r1=992155&r2=992156&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Thu Sep 2
23:26:46 2010
@@ -105,7 +105,7 @@ public final class PigStats {
/**
* JobGraph is an {...@link OperatorPlan} whose members are {...@link
JobStats}
*/
- public static class JobGraph extends BaseOperatorPlan {
+ public static class JobGraph extends BaseOperatorPlan implements
Iterable<JobStats>{
@Override
public String toString() {
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestScalarAliases.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestScalarAliases.java?rev=992156&r1=992155&r2=992156&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestScalarAliases.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestScalarAliases.java Thu Sep 2
23:26:46 2010
@@ -24,7 +24,6 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
-import java.util.Random;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
@@ -45,6 +44,8 @@ public class TestScalarAliases {
@Before
public void setUp() throws Exception{
+ //re-init the variables, so that we can switch between
+ // local and mapreduce modes
FileLocalizer.setInitialized(false);
pigServer = new PigServer(ExecType.LOCAL);
}
@@ -486,6 +487,27 @@ public class TestScalarAliases {
}
+ @Test
+ public void testScalarErrMultipleRowsInInput() throws Exception{
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ String[] input = {
+ "1\t5",
+ "2\t10",
+ "3\t20"
+ };
+ String INPUT_FILE = "table_testScalarAliasesMulRows";
+ Util.createInputFile(cluster, INPUT_FILE, input);
+ pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (a0: long,
a1: double);");
+ pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (a0: long,
a1: double);");
+ pigServer.registerQuery("C = foreach A generate $0, B.$0;");
+ try {
+ pigServer.openIterator("C");
+ fail("exception expected - scalar input has multiple rows");
+ } catch (IOException pe){
+ assertTrue(pe.getCause().getMessage().contains("Scalar has more
than one row in the output"));
+ }
+ }
+
// See PIG-1434
@Test
