Author: rding
Date: Fri Jul 23 18:22:58 2010
New Revision: 967189
URL: http://svn.apache.org/viewvc?rev=967189&view=rev
Log:
PIG-1435: make sure dependent jobs fail when a jon in multiquery fails
Added:
hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=967189&r1=967188&r2=967189&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jul 23 18:22:58 2010
@@ -104,6 +104,9 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1435: make sure dependent jobs fail when a jon in multiquery fails (niraj
+via rding)
+
PIG-1492: DefaultTuple and DefaultMemory understimate their memory footprint
(thejas)
PIG-1409: Fix up javadocs for org.apache.pig.builtin (gates)
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=967189&r1=967188&r2=967189&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Fri Jul 23 18:22:58 2010
@@ -251,10 +251,8 @@ public class MapReduceLauncher extends L
// If we only have one store and that job fail, then we sure
// that the job completely fail, and we shall stop dependent
jobs
for (Job job : jc.getFailedJobs()) {
- List<POStore> sts = jcc.getStores(job);
- if (sts.size()==1) {
- completeFailedJobsInThisRun.add(job);
- }
+ completeFailedJobsInThisRun.add(job);
+ log.info("job " + job.getAssignedJobID() + " has failed!
Stop running all dependent jobs");
}
failedJobs.addAll(jc.getFailedJobs());
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=967189&r1=967188&r2=967189&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Fri Jul 23
18:22:58 2010
@@ -25,12 +25,13 @@ import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Properties;
+import java.util.List;
+import java.util.Iterator;
+
-import org.apache.hadoop.fs.Path;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigRunner.ReturnCode;
-import org.apache.pig.backend.hadoop.datastorage.HPath;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.tools.pigstats.JobStats;
@@ -38,9 +39,8 @@ import org.apache.pig.tools.pigstats.Out
import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
-import org.junit.After;
+import org.apache.pig.experimental.plan.Operator;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -209,7 +209,37 @@ public class TestPigRunner {
Util.deleteFile(cluster, OUTPUT_FILE_2);
}
}
-
+
+ @Test
+ public void MQDepJobFailedTest() throws Exception {
+ final String OUTPUT_FILE_2 = "output2";
+ PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
+ w.println("A = load '" + INPUT_FILE + "' as (name:chararray, a1:int,
a2:int);");
+ w.println("store A into '" + OUTPUT_FILE_2 + "';");
+ w.println("B = FOREACH A GENERATE
org.apache.pig.test.utils.UPPER(name);");
+ w.println("C= order B by $0;");
+ w.println("store C into '" + OUTPUT_FILE + "';");
+ w.close();
+ try {
+ String[] args = { PIG_FILE };
+ PigStats stats = PigRunner.run(args, null);
+ Iterator<JobStats> iter = stats.getJobGraph().iterator();
+ while (iter.hasNext()) {
+ JobStats js=iter.next();
+ if(js.getState().name().equals("FAILED")) {
+ List<Operator> ops=stats.getJobGraph().getSuccessors(js);
+ for(Operator op : ops ) {
+ assertEquals(((JobStats)op).getState().toString(),
"UNKNOWN");
+ }
+ }
+ }
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ Util.deleteFile(cluster, OUTPUT_FILE_2);
+ }
+ }
+
@Test
public void simpleNegativeTest() throws Exception {
PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
Added: hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java?rev=967189&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/utils/UPPER.java Fri Jul 23
18:22:58 2010
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.Tuple;
+
+public class UPPER extends EvalFunc <String> {
+ public String exec(Tuple input) {
+ final int quotient = 5/0;
+ String str="hello";
+ return str;
+ }
+}
