Author: rding Date: Fri Jul 23 18:22:58 2010 New Revision: 967189 URL: http://svn.apache.org/viewvc?rev=967189&view=rev Log: PIG-1435: make sure dependent jobs fail when a jon in multiquery fails
Added: hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=967189&r1=967188&r2=967189&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Jul 23 18:22:58 2010 @@ -104,6 +104,9 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1435: make sure dependent jobs fail when a jon in multiquery fails (niraj +via rding) + PIG-1492: DefaultTuple and DefaultMemory understimate their memory footprint (thejas) PIG-1409: Fix up javadocs for org.apache.pig.builtin (gates) Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=967189&r1=967188&r2=967189&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Fri Jul 23 18:22:58 2010 @@ -251,10 +251,8 @@ public class MapReduceLauncher extends L // If we only have one store and that job fail, then we sure // that the job completely fail, and we shall stop dependent jobs for (Job job : jc.getFailedJobs()) { - List<POStore> sts = jcc.getStores(job); - if (sts.size()==1) { - completeFailedJobsInThisRun.add(job); - } + completeFailedJobsInThisRun.add(job); + log.info("job " + job.getAssignedJobID() + " has failed! Stop running all dependent jobs"); } failedJobs.addAll(jc.getFailedJobs()); } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=967189&r1=967188&r2=967189&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Fri Jul 23 18:22:58 2010 @@ -25,12 +25,13 @@ import java.io.File; import java.io.FileWriter; import java.io.PrintWriter; import java.util.Properties; +import java.util.List; +import java.util.Iterator; + -import org.apache.hadoop.fs.Path; import org.apache.pig.ExecType; import org.apache.pig.PigRunner; import org.apache.pig.PigRunner.ReturnCode; -import org.apache.pig.backend.hadoop.datastorage.HPath; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.io.FileLocalizer; import org.apache.pig.tools.pigstats.JobStats; @@ -38,9 +39,8 @@ import org.apache.pig.tools.pigstats.Out import org.apache.pig.tools.pigstats.PigProgressNotificationListener; import org.apache.pig.tools.pigstats.PigStats; import org.apache.pig.tools.pigstats.PigStatsUtil; -import org.junit.After; +import org.apache.pig.experimental.plan.Operator; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -209,7 +209,37 @@ public class TestPigRunner { Util.deleteFile(cluster, OUTPUT_FILE_2); } } - + + @Test + public void MQDepJobFailedTest() throws Exception { + final String OUTPUT_FILE_2 = "output2"; + PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE)); + w.println("A = load '" + INPUT_FILE + "' as (name:chararray, a1:int, a2:int);"); + w.println("store A into '" + OUTPUT_FILE_2 + "';"); + w.println("B = FOREACH A GENERATE org.apache.pig.test.utils.UPPER(name);"); + w.println("C= order B by $0;"); + w.println("store C into '" + OUTPUT_FILE + "';"); + w.close(); + try { + String[] args = { PIG_FILE }; + PigStats stats = PigRunner.run(args, null); + Iterator<JobStats> iter = stats.getJobGraph().iterator(); + while (iter.hasNext()) { + JobStats js=iter.next(); + if(js.getState().name().equals("FAILED")) { + List<Operator> ops=stats.getJobGraph().getSuccessors(js); + for(Operator op : ops ) { + assertEquals(((JobStats)op).getState().toString(), "UNKNOWN"); + } + } + } + } finally { + new File(PIG_FILE).delete(); + Util.deleteFile(cluster, OUTPUT_FILE); + Util.deleteFile(cluster, OUTPUT_FILE_2); + } + } + @Test public void simpleNegativeTest() throws Exception { PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE)); Added: hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java?rev=967189&view=auto ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java (added) +++ hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java Fri Jul 23 18:22:58 2010 @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pig.test.utils; + +import org.apache.pig.EvalFunc; +import org.apache.pig.data.Tuple; + +public class UPPER extends EvalFunc <String> { + public String exec(Tuple input) { + final int quotient = 5/0; + String str="hello"; + return str; + } +}