TEZ-3017. HistoryACLManager does not have a close method for cleanup (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9816a49b Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9816a49b Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9816a49b Branch: refs/heads/TEZ-2980 Commit: 9816a49bba0c89651f9b5bb66ac20f4eb0364ef2 Parents: 5334d62 Author: Bikas Saha <[email protected]> Authored: Tue Dec 29 01:54:00 2015 -0800 Committer: Bikas Saha <[email protected]> Committed: Tue Dec 29 01:54:00 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../src/main/java/org/apache/tez/client/TezClient.java | 4 ++++ .../tez/common/security/HistoryACLPolicyManager.java | 5 ++++- .../test/java/org/apache/tez/client/TestTezClient.java | 6 +++++- .../dag/history/ats/acls/ATSHistoryACLPolicyManager.java | 11 ++++++++--- .../tez/dag/history/ats/acls/TestATSHistoryWithACLs.java | 9 ++++++--- .../history/logging/ats/ATSHistoryLoggingService.java | 3 +++ 7 files changed, 32 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 546c3fd..ce35fd1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,6 +11,7 @@ INCOMPATIBLE CHANGES TEZ-2972. Avoid task rescheduling when a node turns unhealthy ALL CHANGES: + TEZ-3017. HistoryACLManager does not have a close method for cleanup TEZ-2914. Ability to limit vertex concurrency TEZ-3011. Link Vertex Name in Dag Tasks/Task Attempts to Vertex TEZ-3006. Remove unused import in TestHistoryParser. @@ -300,6 +301,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES + TEZ-3017. HistoryACLManager does not have a close method for cleanup TEZ-2914. Ability to limit vertex concurrency TEZ-2918. Make progress notifications in IOs TEZ-2952. NPE in TestOnFileUnorderedKVOutput http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index ff131e0..fc98d1a 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -558,6 +558,10 @@ public class TezClient { */ public synchronized void stop() throws TezException, IOException { try { + if (historyACLPolicyManager != null) { + historyACLPolicyManager.close(); + } + if (sessionStarted) { LOG.info("Shutting down Tez Session" + ", sessionName=" + clientName http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java index dea89cc..fc0f57c 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java @@ -69,5 +69,8 @@ public interface HistoryACLPolicyManager extends Configurable { public void updateTimelineEntityDomain(Object timelineEntity, String domainId); - + /** + * Call this to stop and clean up + */ + public void close(); } http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index 9adb9bd..c2531c6 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.tez.common.counters.LimitExceededException; import org.apache.tez.common.counters.Limits; import org.apache.tez.common.counters.TezCounters; +import org.apache.tez.common.security.HistoryACLPolicyManager; import org.apache.tez.dag.api.DAG; import org.apache.tez.dag.api.PreWarmVertex; import org.apache.tez.dag.api.ProcessorDescriptor; @@ -162,10 +163,12 @@ public class TestTezClient { LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null); - + HistoryACLPolicyManager mockAcl = mock(HistoryACLPolicyManager.class); + ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); + client.setUpHistoryAclManager(mockAcl); client.start(); verify(client.mockYarnClient, times(1)).init((Configuration)any()); verify(client.mockYarnClient, times(1)).start(); @@ -263,6 +266,7 @@ public class TestTezClient { (ShutdownSessionRequestProto) any()); } verify(client.mockYarnClient, times(1)).stop(); + verify(mockAcl, times(1)).close(); } @Test (timeout=5000) http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java index 3fa3db6..91ffe7b 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java @@ -24,6 +24,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.hadoop.service.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -32,7 +33,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.tez.common.security.ACLConfigurationParser; import org.apache.tez.common.security.ACLManager; import org.apache.tez.common.security.ACLType; @@ -43,8 +43,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezUncheckedException; -import com.google.common.annotations.VisibleForTesting; - public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager { private final static Logger LOG = LoggerFactory.getLogger(ATSHistoryACLPolicyManager.class); @@ -250,4 +248,11 @@ public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager { entity.setDomainId(domainId); } + @Override + public void close() { + if (timelineClient != null && timelineClient.isInState(Service.STATE.STARTED)) { + timelineClient.stop(); + } + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java index eaf24d3..512913d 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.*; import java.io.IOException; import java.util.Collection; @@ -74,9 +75,6 @@ import com.sun.jersey.api.client.WebResource; import org.mockito.Matchers; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.doThrow; - public class TestATSHistoryWithACLs { private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryWithACLs.class); @@ -385,7 +383,9 @@ public class TestATSHistoryWithACLs { } dagLogging = dag2.getDagConf().get(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED); Assert.assertNull(dagLogging); + myAclPolicyManager.timelineClient = spy(myAclPolicyManager.timelineClient); tezSession.stop(); + verify(myAclPolicyManager.timelineClient, times(1)).stop(); } /** @@ -465,7 +465,10 @@ public class TestATSHistoryWithACLs { } dagLogging = dag2.getDagConf().get(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED); Assert.assertNull(dagLogging); + myAclPolicyManager.timelineClient = spy(myAclPolicyManager.timelineClient); tezClient.stop(); + verify(myAclPolicyManager.timelineClient, times(1)).stop(); + } /** * Test Disable Logging for all dags in a session http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java index 6ea21e2..a66da24 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java @@ -243,6 +243,9 @@ public class ATSHistoryLoggingService extends HistoryLoggingService { + ", eventQueueBacklog=" + eventQueue.size()); } timelineClient.stop(); + if (historyACLPolicyManager != null) { + historyACLPolicyManager.close(); + } } private void getEventBatch(List<DAGHistoryEvent> events) throws InterruptedException {
