tez git commit: TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. (mingma)
Repository: tez Updated Branches: refs/heads/branch-0.8 550fa4323 -> 91bc21a3e TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. (mingma) (cherry picked from commit 5c2f893ab850e077f1c007bb7291c7abe9914b4d) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/91bc21a3 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/91bc21a3 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/91bc21a3 Branch: refs/heads/branch-0.8 Commit: 91bc21a3e9710fc2c7edb696b15245999ddac811 Parents: 550fa43 Author: Ming Ma Authored: Thu Sep 22 15:22:21 2016 -0700 Committer: Ming Ma Committed: Thu Sep 22 15:25:16 2016 -0700 -- CHANGES.txt | 1 + .../main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java | 6 +++--- .../tez/dag/history/events/VertexConfigurationDoneEvent.java | 1 + .../dag/history/events/TestHistoryEventsProtoConversion.java | 7 ++- 4 files changed, 11 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/tez/blob/91bc21a3/CHANGES.txt -- diff --git a/CHANGES.txt b/CHANGES.txt index 5c85926..576435c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. TEZ-3000. Fix TestContainerReuse. TEZ-3436. Check input and output count before start in MapProcessor. TEZ-3426. Second AM attempt launched for session mode and recovery disabled for certain cases http://git-wip-us.apache.org/repos/asf/tez/blob/91bc21a3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java -- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 01bca8f..b58e2c2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -2578,10 +2578,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl && recoveryData.shouldSkipInit()) { // Replace the original VertexManager with NoOpVertexManager if the reconfiguration is done in the last AM attempt VertexConfigurationDoneEvent reconfigureDoneEvent = recoveryData.getVertexConfigurationDoneEvent(); - if (LOG.isDebugEnabled()) { -LOG.debug("VertexManager reconfiguration is done in the last AM Attempt" + if (LOG.isInfoEnabled()) { +LOG.info("VertexManager reconfiguration is done in the last AM Attempt" + ", use NoOpVertexManager to replace it, vertexId=" + logIdentifier); -LOG.debug("VertexReconfigureDoneEvent=" + reconfigureDoneEvent); +LOG.info("VertexReconfigureDoneEvent=" + reconfigureDoneEvent); } ByteArrayOutputStream out = new ByteArrayOutputStream(); try { http://git-wip-us.apache.org/repos/asf/tez/blob/91bc21a3/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java -- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java index 4ad1c63..137342c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java @@ -58,6 +58,7 @@ public class VertexConfigurationDoneEvent implements HistoryEvent { boolean setParallelismCalledFlag) { super(); this.vertexID = vertexID; +this.reconfigureDoneTime = reconfigureDoneTime; this.numTasks = numTasks; this.vertexLocationHint = vertexLocationHint; this.sourceEdgeProperties = sourceEdgeProperties; http://git-wip-us.apache.org/repos/asf/tez/blob/91bc21a3/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java -- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java index 67a927e..3d29a5d 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java @@ -335,11 +335,16 @@ public class TestHistoryEventsProtoConversion { DataS
tez git commit: TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. (mingma)
Repository: tez Updated Branches: refs/heads/master b728dbb2b -> 5c2f893ab TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. (mingma) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5c2f893a Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5c2f893a Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5c2f893a Branch: refs/heads/master Commit: 5c2f893ab850e077f1c007bb7291c7abe9914b4d Parents: b728dbb Author: Ming Ma Authored: Thu Sep 22 15:22:21 2016 -0700 Committer: Ming Ma Committed: Thu Sep 22 15:22:21 2016 -0700 -- CHANGES.txt | 2 ++ .../main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java | 6 +++--- .../tez/dag/history/events/VertexConfigurationDoneEvent.java | 1 + .../dag/history/events/TestHistoryEventsProtoConversion.java | 7 ++- 4 files changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/tez/blob/5c2f893a/CHANGES.txt -- diff --git a/CHANGES.txt b/CHANGES.txt index 9a24eac..db57a61 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. TEZ-3000. Fix TestContainerReuse. TEZ-3436. Check input and output count before start in MapProcessor. TEZ-3163. Reuse and tune Inflaters and Deflaters to speed DME processing @@ -112,6 +113,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. TEZ-3000. Fix TestContainerReuse. TEZ-3436. Check input and output count before start in MapProcessor. TEZ-3426. Second AM attempt launched for session mode and recovery disabled for certain cases http://git-wip-us.apache.org/repos/asf/tez/blob/5c2f893a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java -- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 1dd7756..38ef89f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -2578,10 +2578,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl && recoveryData.shouldSkipInit()) { // Replace the original VertexManager with NoOpVertexManager if the reconfiguration is done in the last AM attempt VertexConfigurationDoneEvent reconfigureDoneEvent = recoveryData.getVertexConfigurationDoneEvent(); - if (LOG.isDebugEnabled()) { -LOG.debug("VertexManager reconfiguration is done in the last AM Attempt" + if (LOG.isInfoEnabled()) { +LOG.info("VertexManager reconfiguration is done in the last AM Attempt" + ", use NoOpVertexManager to replace it, vertexId=" + logIdentifier); -LOG.debug("VertexReconfigureDoneEvent=" + reconfigureDoneEvent); +LOG.info("VertexReconfigureDoneEvent=" + reconfigureDoneEvent); } NonSyncByteArrayOutputStream out = new NonSyncByteArrayOutputStream(); try { http://git-wip-us.apache.org/repos/asf/tez/blob/5c2f893a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java -- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java index 4ad1c63..137342c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java @@ -58,6 +58,7 @@ public class VertexConfigurationDoneEvent implements HistoryEvent { boolean setParallelismCalledFlag) { super(); this.vertexID = vertexID; +this.reconfigureDoneTime = reconfigureDoneTime; this.numTasks = numTasks; this.vertexLocationHint = vertexLocationHint; this.sourceEdgeProperties = sourceEdgeProperties; http://git-wip-us.apache.org/repos/asf/tez/blob/5c2f893a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java -- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java index 67a927e..3d29a5d 100644 --- a/tez-dag/src/test/ja