Author: gates
Date: Fri Sep 26 17:00:03 2008
New Revision: 699534
URL: http://svn.apache.org/viewvc?rev=699534&view=rev
Log:
PIG-457 This just addresses the issue of getting false error messages on
success. Launchers changed to send any error messages from hadoop to debug if
the
job succeeded.
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java?rev=699534&r1=699533&r2=699534&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
Fri Sep 26 17:00:03 2008
@@ -85,13 +85,13 @@
return (int)(Math.ceil(prog)) == (int)1;
}
- protected void getStats(Job job, JobClient jobClient) throws IOException{
+ protected void getStats(Job job, JobClient jobClient, boolean errNotDbg)
throws IOException{
JobID MRJobID = job.getAssignedJobID();
TaskReport[] mapRep = jobClient.getMapTaskReports(MRJobID);
- getErrorMessages(mapRep, "map");
+ getErrorMessages(mapRep, "map", errNotDbg);
totalHadoopTimeSpent += computeTimeSpent(mapRep);
TaskReport[] redRep = jobClient.getReduceTaskReports(MRJobID);
- getErrorMessages(redRep, "reduce");
+ getErrorMessages(redRep, "reduce", errNotDbg);
totalHadoopTimeSpent += computeTimeSpent(mapRep);
}
@@ -103,13 +103,18 @@
return timeSpent;
}
- protected void getErrorMessages(TaskReport reports[], String type)
+ protected void getErrorMessages(TaskReport reports[], String type, boolean
errNotDbg)
{
for (int i = 0; i < reports.length; i++) {
String msgs[] = reports[i].getDiagnostics();
for (int j = 0; j < msgs.length; j++) {
- log.error("Error message from task (" + type + ") " +
- reports[i].getTaskID() + msgs[j]);
+ if (errNotDbg) {
+ log.error("Error message from task (" + type + ") " +
+ reports[i].getTaskID() + msgs[j]);
+ } else {
+ log.debug("Error message from task (" + type + ") " +
+ reports[i].getTaskID() + msgs[j]);
+ }
}
}
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java?rev=699534&r1=699533&r2=699534&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
Fri Sep 26 17:00:03 2008
@@ -20,6 +20,7 @@
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRPrinter;
+import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRStreamHandler;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
@@ -70,15 +71,16 @@
log.error("Map reduce job failed");
for (Job fj : failedJobs) {
log.error(fj.getMessage());
- getStats(fj, jobClient);
+ getStats(fj, jobClient, true);
}
+ jc.stop();
return false;
}
List<Job> succJobs = jc.getSuccessfulJobs();
if(succJobs!=null)
for(Job job : succJobs){
- getStats(job,jobClient);
+ getStats(job,jobClient, false);
}
jc.stop();
@@ -111,6 +113,11 @@
CombinerOptimizer co = new CombinerOptimizer(plan);
co.visit();
}
+
+ // check whether stream operator is present
+ MRStreamHandler checker = new MRStreamHandler(plan);
+ checker.visit();
+
// figure out the type of the key for the map plan
// this is needed when the key is null to create
// an appropriate NullableXXXWritable object
@@ -168,13 +175,13 @@
throw new ExecException(
"Something terribly wrong with Job Control.");
for (Job job : failedJobs) {
- getStats(job, jobClient);
+ getStats(job, jobClient, true);
}
}
List<Job> succJobs = jc.getSuccessfulJobs();
if (succJobs != null)
for (Job job : succJobs) {
- getStats(job, jobClient);
+ getStats(job, jobClient, false);
}
jc.stop();
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=699534&r1=699533&r2=699534&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Fri Sep 26 17:00:03 2008
@@ -88,15 +88,16 @@
log.error("Map reduce job failed");
for (Job fj : failedJobs) {
log.error(fj.getMessage());
- getStats(fj, jobClient);
+ getStats(fj, jobClient, true);
}
+ jc.stop();
return false;
}
List<Job> succJobs = jc.getSuccessfulJobs();
if(succJobs!=null)
for(Job job : succJobs){
- getStats(job,jobClient);
+ getStats(job,jobClient, false);
}
jc.stop();
