YARN-7962. Race Condition When Stopping DelegationTokenRenewer causes RM crash during failover. (BELUGA BEHR via wangda)
Change-Id: I617e2645f60a57080058ad5f06af860fb3f682c8 (cherry picked from commit 931f78718f3a09775bfa1f9a952c069c416d0914) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cee85d8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cee85d8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cee85d8 Branch: refs/heads/branch-3.1 Commit: 1cee85d8c42e97c8df6fbc66cd0eb3022ed854bb Parents: cc460fe Author: Wangda Tan <[email protected]> Authored: Fri Jun 1 14:00:18 2018 -0700 Committer: Wangda Tan <[email protected]> Committed: Fri Jun 1 15:47:51 2018 -0700 ---------------------------------------------------------------------- .../security/DelegationTokenRenewer.java | 10 +++++++- .../security/TestDelegationTokenRenewer.java | 24 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cee85d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 220787c..a9f8cd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -209,7 +209,15 @@ public class DelegationTokenRenewer extends AbstractService { } appTokens.clear(); allTokens.clear(); - this.renewerService.shutdown(); + + serviceStateLock.writeLock().lock(); + try { + isServiceStarted = false; + this.renewerService.shutdown(); + } finally { + serviceStateLock.writeLock().unlock(); + } + dtCancelThread.interrupt(); try { dtCancelThread.join(1000); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cee85d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 8351860..9b2c0b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1420,4 +1420,28 @@ public class TestDelegationTokenRenewer { delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); assertNull(mockDttr.timerTask); } + + /** + * Test that the DelegationTokenRenewer class can gracefully handle + * interactions that occur when it has been stopped. + */ + @Test + public void testShutDown() { + DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter); + RMContext mockContext = mock(RMContext.class); + when(mockContext.getSystemCredentialsForApps()).thenReturn( + new ConcurrentHashMap<ApplicationId, ByteBuffer>()); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + dtr.setRMContext(mockContext); + when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); + dtr.init(conf); + dtr.start(); + delegationTokenRenewer.stop(); + delegationTokenRenewer.applicationFinished( + BuilderUtils.newApplicationId(0, 1)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
