Repository: crunch Updated Branches: refs/heads/apache-crunch-0.8 74c18d453 -> 5c433c8f4
CRUNCH-392: Correctly set failure status on any kind of Exception Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/5c433c8f Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/5c433c8f Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/5c433c8f Branch: refs/heads/apache-crunch-0.8 Commit: 5c433c8f4d114cd8ef8008fc27cdda41c17edff7 Parents: 74c18d4 Author: Josh Wills <[email protected]> Authored: Mon May 12 15:22:05 2014 -0700 Committer: Josh Wills <[email protected]> Committed: Tue May 13 14:52:54 2014 -0700 ---------------------------------------------------------------------- .../apache/crunch/impl/dist/collect/FailIT.java | 74 ++++++++++++++++++++ .../apache/crunch/impl/mr/exec/MRExecutor.java | 2 +- 2 files changed, 75 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/5c433c8f/crunch-core/src/it/java/org/apache/crunch/impl/dist/collect/FailIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/impl/dist/collect/FailIT.java b/crunch-core/src/it/java/org/apache/crunch/impl/dist/collect/FailIT.java new file mode 100644 index 0000000..d0d7677 --- /dev/null +++ b/crunch-core/src/it/java/org/apache/crunch/impl/dist/collect/FailIT.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.impl.dist.collect; + +import org.apache.crunch.MapFn; +import org.apache.crunch.PCollection; +import org.apache.crunch.Pipeline; +import org.apache.crunch.PipelineExecution; +import org.apache.crunch.impl.mr.MRPipeline; +import org.apache.crunch.test.CrunchTestSupport; +import org.apache.crunch.types.writable.Writables; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class FailIT extends CrunchTestSupport { + + static class InverseFn extends MapFn<String, Integer> { + @Override + public Integer map(String input) { + int c = 0; + return 1 / c; + } + }; + + @Test + public void testKill() throws Exception { + Pipeline pipeline = new MRPipeline(FailIT.class, tempDir.getDefaultConfiguration()); + PCollection<String> p = pipeline.readTextFile(tempDir.copyResourceFileName("shakes.txt")); + PCollection<Integer> result = p.parallelDo(new InverseFn(), Writables.ints()); + result.cache(); + + PipelineExecution execution = pipeline.runAsync(); + + while (!execution.isDone() && !execution.isCancelled() + && execution.getStatus() != PipelineExecution.Status.FAILED + && execution.getResult() == null) { + try { + Thread.sleep(1000); + System.out.println("Job Status: " + execution.getStatus().toString()); + } catch (InterruptedException e) { + System.err.println("ABORTING"); + e.printStackTrace(); + try { + execution.kill(); + execution.waitUntilDone(); + } catch (InterruptedException e1) { + throw new RuntimeException(e1); + } + throw new RuntimeException(e); + } + } + System.out.println("Finished running job."); + assertEquals(PipelineExecution.Status.FAILED, execution.getStatus()); + } + +} http://git-wip-us.apache.org/repos/asf/crunch/blob/5c433c8f/crunch-core/src/main/java/org/apache/crunch/impl/mr/exec/MRExecutor.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/exec/MRExecutor.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/exec/MRExecutor.java index 1137498..eb46ab0 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/exec/MRExecutor.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/exec/MRExecutor.java @@ -166,7 +166,7 @@ public class MRExecutor extends AbstractFuture<PipelineResult> implements MRPipe } } catch (InterruptedException e) { throw new AssertionError(e); // Nobody should interrupt us. - } catch (IOException e) { + } catch (Exception e) { LOG.error("Pipeline failed due to exception", e); status.set(Status.FAILED); setException(e);
