TEZ-137 VertexHistoryEvents are using VertexStatus.state instead of VertexImpl.state (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/3bb954bd Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/3bb954bd Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/3bb954bd Branch: refs/heads/master Commit: 3bb954bd9153f1b6b7e394776d9169cd99acde1b Parents: a2a7261 Author: Bikas Saha <[email protected]> Authored: Tue May 28 13:33:52 2013 -0700 Committer: Bikas Saha <[email protected]> Committed: Tue May 28 13:33:52 2013 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/tez/client/TezClient.java | 3 - .../hadoop/mapred/MRVertexOutputCommitter.java | 19 +++++- .../apache/tez/dag/api/committer/DAGStatus.java | 39 ----------- .../api/committer/NullVertexOutputCommitter.java | 4 +- .../dag/api/committer/VertexOutputCommitter.java | 1 + .../apache/tez/dag/api/committer/VertexStatus.java | 51 --------------- .../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 2 +- .../apache/tez/dag/app/dag/impl/VertexImpl.java | 12 ++-- .../tez/dag/history/events/DAGFinishedEvent.java | 2 +- .../dag/history/events/VertexFinishedEvent.java | 2 +- .../tez/dag/app/dag/impl/TestVertexImpl.java | 4 +- 11 files changed, 32 insertions(+), 107 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java index f794014..73c97b3 100644 --- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; -import org.apache.tez.dag.api.TezRemoteException; import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.VertexStatus; @@ -45,7 +44,6 @@ public class TezClient { public DAGClient getDAGClient(String appIdStr) throws IOException, TezException { try { - System.out.println("Fetching app: " + appIdStr); ApplicationId appId = ConverterUtils.toApplicationId(appIdStr); YarnClient yarnClient = new YarnClientImpl(); yarnClient.init(conf); @@ -60,7 +58,6 @@ public class TezClient { } public DAGClient getDAGClient(String host, int port) throws IOException { - System.out.println("App host port: " + host + ":" + port); InetSocketAddress addr = new InetSocketAddress(host, port); DAGClient dagClient; dagClient = new DAGClientRPCImpl(1, addr, conf); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java index 7c886d6..171c244 100644 --- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java +++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java @@ -33,9 +33,10 @@ import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.YarnException; +import org.apache.tez.dag.api.TezException; +import org.apache.tez.dag.api.client.VertexStatus; import org.apache.tez.dag.api.committer.VertexContext; import org.apache.tez.dag.api.committer.VertexOutputCommitter; -import org.apache.tez.dag.api.committer.VertexStatus; import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.records.TezTaskID; import org.apache.tez.dag.utils.TezBuilderUtils; @@ -102,7 +103,21 @@ public class MRVertexOutputCommitter extends VertexOutputCommitter { } private State getJobStateFromVertexStatusState(VertexStatus.State state) { - return JobStatus.State.valueOf(state.name()); + switch(state) { + case INITED: + return JobStatus.State.PREP; + case RUNNING: + return JobStatus.State.RUNNING; + case SUCCEEDED: + return JobStatus.State.SUCCEEDED; + case KILLED: + return JobStatus.State.KILLED; + case FAILED: + case ERROR: + return JobStatus.State.FAILED; + default: + throw new TezException("Unknown VertexStatus.State: " + state); + } } @Override http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java deleted file mode 100644 index 587e18f..0000000 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java +++ /dev/null @@ -1,39 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.tez.dag.api.committer; - -public class DAGStatus { - public static enum State { - RUNNING(1), - SUCCEEDED(2), - FAILED(3), - PREP(4), - KILLED(5); - - int value; - - State(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - - }; -} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java index c284ab5..08b2d4a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java @@ -20,7 +20,7 @@ package org.apache.tez.dag.api.committer; import java.io.IOException; -import org.apache.tez.dag.api.committer.VertexStatus.State; +import org.apache.tez.dag.api.client.VertexStatus; public class NullVertexOutputCommitter extends VertexOutputCommitter { @@ -40,7 +40,7 @@ public class NullVertexOutputCommitter extends VertexOutputCommitter { } @Override - public void abortVertex(State finalState) throws IOException { + public void abortVertex(VertexStatus.State finalState) throws IOException { // Nothing to do } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java index b814d0e..34c6e89 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.tez.dag.api.client.VertexStatus; @Public @Unstable http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java deleted file mode 100644 index a78027a..0000000 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java +++ /dev/null @@ -1,51 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.tez.dag.api.committer; - -public class VertexStatus { - - public static enum State { - RUNNING(1), - SUCCEEDED(2), - FAILED(3), - PREP(4), // TODO change to INITING - KILLED(5); - - int value; - - State(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - - }; - - private State runState; - - public synchronized State getState() { - return runState; - } - - public synchronized void setState(State state) { - this.runState = state; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java index 4c63d8f..cc6699d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java @@ -59,10 +59,10 @@ import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.VertexLocationHint; +import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.DAGStatusBuilder; import org.apache.tez.dag.api.client.ProgressBuilder; import org.apache.tez.dag.api.client.VertexStatusBuilder; -import org.apache.tez.dag.api.committer.DAGStatus; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.app.AppContext; import org.apache.tez.dag.app.TaskAttemptListener; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index ce4c609..e34382f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -61,15 +61,15 @@ import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern; import org.apache.tez.dag.api.EdgeProperty; +import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint; import org.apache.tez.dag.api.client.ProgressBuilder; +import org.apache.tez.dag.api.client.VertexStatus; import org.apache.tez.dag.api.client.VertexStatusBuilder; import org.apache.tez.dag.api.committer.NullVertexOutputCommitter; import org.apache.tez.dag.api.committer.VertexContext; import org.apache.tez.dag.api.committer.VertexOutputCommitter; -import org.apache.tez.dag.api.committer.VertexStatus; -import org.apache.tez.dag.api.committer.VertexStatus.State; import org.apache.tez.dag.app.AppContext; import org.apache.tez.dag.app.TaskAttemptListener; import org.apache.tez.dag.app.TaskHeartbeatHandler; @@ -744,23 +744,25 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, case KILL_WAIT: eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState)); - logJobHistoryVertexFailedEvent(State.KILLED); + logJobHistoryVertexFailedEvent(VertexStatus.State.KILLED); break; case ERROR: eventHandler.handle(new DAGEvent(getDAGId(), DAGEventType.INTERNAL_ERROR)); - logJobHistoryVertexFailedEvent(State.FAILED); + logJobHistoryVertexFailedEvent(VertexStatus.State.FAILED); break; case FAILED: eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState)); - logJobHistoryVertexFailedEvent(State.FAILED); + logJobHistoryVertexFailedEvent(VertexStatus.State.FAILED); break; case SUCCEEDED: eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState)); logJobHistoryVertexFinishedEvent(); break; + default: + throw new TezException("Unexpected VertexState: " + finalState); } return finalState; } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java index 5dea9a7..8794cea 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java @@ -18,7 +18,7 @@ package org.apache.tez.dag.history.events; -import org.apache.tez.dag.api.committer.DAGStatus; +import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.avro.DAGFinished; import org.apache.tez.dag.history.avro.HistoryEventType; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java index 91c209f..7e86c66 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java @@ -18,7 +18,7 @@ package org.apache.tez.dag.history.events; -import org.apache.tez.dag.api.committer.VertexStatus; +import org.apache.tez.dag.api.client.VertexStatus; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.avro.HistoryEventType; import org.apache.tez.dag.history.avro.VertexFinished; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java index f78a7b8..e4a1040 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java @@ -50,10 +50,10 @@ import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.VertexLocationHint; +import org.apache.tez.dag.api.client.VertexStatus; import org.apache.tez.dag.api.committer.NullVertexOutputCommitter; import org.apache.tez.dag.api.committer.VertexContext; import org.apache.tez.dag.api.committer.VertexOutputCommitter; -import org.apache.tez.dag.api.committer.VertexStatus.State; import org.apache.tez.dag.app.AppContext; import org.apache.tez.dag.app.TaskAttemptListener; import org.apache.tez.dag.app.TaskHeartbeatHandler; @@ -147,7 +147,7 @@ public class TestVertexImpl { } @Override - public void abortVertex(State finalState) throws IOException { + public void abortVertex(VertexStatus.State finalState) throws IOException { ++abortCounter; if (throwErrorOnAbort) { throw new IOException("I can throwz exceptions in abort");
