Author: gates Date: Fri Sep 26 17:00:03 2008 New Revision: 699534 URL: http://svn.apache.org/viewvc?rev=699534&view=rev Log: PIG-457 This just addresses the issue of getting false error messages on success. Launchers changed to send any error messages from hadoop to debug if the job succeeded.
Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java?rev=699534&r1=699533&r2=699534&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java Fri Sep 26 17:00:03 2008 @@ -85,13 +85,13 @@ return (int)(Math.ceil(prog)) == (int)1; } - protected void getStats(Job job, JobClient jobClient) throws IOException{ + protected void getStats(Job job, JobClient jobClient, boolean errNotDbg) throws IOException{ JobID MRJobID = job.getAssignedJobID(); TaskReport[] mapRep = jobClient.getMapTaskReports(MRJobID); - getErrorMessages(mapRep, "map"); + getErrorMessages(mapRep, "map", errNotDbg); totalHadoopTimeSpent += computeTimeSpent(mapRep); TaskReport[] redRep = jobClient.getReduceTaskReports(MRJobID); - getErrorMessages(redRep, "reduce"); + getErrorMessages(redRep, "reduce", errNotDbg); totalHadoopTimeSpent += computeTimeSpent(mapRep); } @@ -103,13 +103,18 @@ return timeSpent; } - protected void getErrorMessages(TaskReport reports[], String type) + protected void getErrorMessages(TaskReport reports[], String type, boolean errNotDbg) { for (int i = 0; i < reports.length; i++) { String msgs[] = reports[i].getDiagnostics(); for (int j = 0; j < msgs.length; j++) { - log.error("Error message from task (" + type + ") " + - reports[i].getTaskID() + msgs[j]); + if (errNotDbg) { + log.error("Error message from task (" + type + ") " + + reports[i].getTaskID() + msgs[j]); + } else { + log.debug("Error message from task (" + type + ") " + + reports[i].getTaskID() + msgs[j]); + } } } } Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java?rev=699534&r1=699533&r2=699534&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java Fri Sep 26 17:00:03 2008 @@ -20,6 +20,7 @@ import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRPrinter; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRStreamHandler; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.impl.plan.PlanException; import org.apache.pig.impl.plan.VisitorException; @@ -70,15 +71,16 @@ log.error("Map reduce job failed"); for (Job fj : failedJobs) { log.error(fj.getMessage()); - getStats(fj, jobClient); + getStats(fj, jobClient, true); } + jc.stop(); return false; } List<Job> succJobs = jc.getSuccessfulJobs(); if(succJobs!=null) for(Job job : succJobs){ - getStats(job,jobClient); + getStats(job,jobClient, false); } jc.stop(); @@ -111,6 +113,11 @@ CombinerOptimizer co = new CombinerOptimizer(plan); co.visit(); } + + // check whether stream operator is present + MRStreamHandler checker = new MRStreamHandler(plan); + checker.visit(); + // figure out the type of the key for the map plan // this is needed when the key is null to create // an appropriate NullableXXXWritable object @@ -168,13 +175,13 @@ throw new ExecException( "Something terribly wrong with Job Control."); for (Job job : failedJobs) { - getStats(job, jobClient); + getStats(job, jobClient, true); } } List<Job> succJobs = jc.getSuccessfulJobs(); if (succJobs != null) for (Job job : succJobs) { - getStats(job, jobClient); + getStats(job, jobClient, false); } jc.stop(); Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=699534&r1=699533&r2=699534&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Fri Sep 26 17:00:03 2008 @@ -88,15 +88,16 @@ log.error("Map reduce job failed"); for (Job fj : failedJobs) { log.error(fj.getMessage()); - getStats(fj, jobClient); + getStats(fj, jobClient, true); } + jc.stop(); return false; } List<Job> succJobs = jc.getSuccessfulJobs(); if(succJobs!=null) for(Job job : succJobs){ - getStats(job,jobClient); + getStats(job,jobClient, false); } jc.stop();