Repository: hadoop Updated Branches: refs/heads/branch-2.7 2c51bbda5 -> ece1b1645
YARN-4321. Incessant retries if NoAuthException is thrown by Zookeeper in non HA mode. Contributed by Varun Saxena Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ece1b164 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ece1b164 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ece1b164 Branch: refs/heads/branch-2.7 Commit: ece1b1645bc6e7b526b817f55288a13146cc3b30 Parents: 2c51bbd Author: Jian He <[email protected]> Authored: Mon Nov 2 11:22:30 2015 -0800 Committer: Jian He <[email protected]> Committed: Mon Nov 2 11:22:30 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../recovery/ZKRMStateStore.java | 2 + .../recovery/TestZKRMStateStore.java | 47 +++++++++++++++++--- 3 files changed, 46 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ece1b164/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c32100b..a3e5010 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -112,6 +112,9 @@ Release 2.7.2 - UNRELEASED YARN-4320. TestJobHistoryEventHandler fails as AHS in MiniYarnCluster no longer binds to default port 8188. (Varun Saxena via ozawa) + YARN-4321. Incessant retries if NoAuthException is thrown by Zookeeper in non + HA mode. (Varun Saxena via jianhe) + Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ece1b164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 3eeb895..543d891 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -1174,6 +1174,8 @@ public class ZKRMStateStore extends RMStateStore { // another RM becoming active. Even if not, // it is safer to assume we have been fenced throw new StoreFencedException(); + } else { + throw nae; } } catch (KeeperException ke) { if (ke.code() == Code.NODEEXISTS) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ece1b164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 991c35f..019915c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -19,12 +19,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import java.io.IOException; -import java.util.HashMap; import java.util.List; import javax.crypto.SecretKey; @@ -37,6 +40,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.service.Service; +import org.apache.hadoop.service.ServiceStateException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -49,6 +53,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; @@ -61,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.data.Stat; import org.junit.Assert; import org.junit.Test; @@ -74,7 +80,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { ZooKeeper client; TestZKRMStateStoreInternal store; - String workingZnode; + String workingZnode = "/jira/issue/3077/rmstore"; class TestZKRMStateStoreInternal extends ZKRMStateStore { @@ -104,16 +110,27 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { } } - public RMStateStore getRMStateStore() throws Exception { + public RMStateStore getRMStateStore(ZooKeeper zk) throws Exception { YarnConfiguration conf = new YarnConfiguration(); - workingZnode = "/jira/issue/3077/rmstore"; conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode); - this.client = createClient(); + if (null == zk) { + this.client = createClient(); + } else { + this.client = zk; + } this.store = new TestZKRMStateStoreInternal(conf, workingZnode); return this.store; } + public String getWorkingZNode() { + return workingZnode; + } + + public RMStateStore getRMStateStore() throws Exception { + return getRMStateStore(null); + } + @Override public boolean isFinalStateValid() throws Exception { List<String> nodes = client.getChildren(store.znodeWorkingPath, false); @@ -262,7 +279,25 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { HAServiceProtocol.HAServiceState.ACTIVE, rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); } - + + @Test + public void testNoAuthExceptionInNonHAMode() throws Exception { + TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); + String appRoot = zkTester.getWorkingZNode() + "/ZKRMStateRoot/RMAppRoot" ; + ZooKeeper zk = spy(createClient()); + doThrow(new KeeperException.NoAuthException()).when(zk). + create(appRoot, null, RMZKUtils.getZKAcls(new Configuration()), + CreateMode.PERSISTENT); + try { + zkTester.getRMStateStore(zk); + fail("Expected exception to be thrown"); + } catch(ServiceStateException e) { + assertNotNull(e.getCause()); + assertTrue("Expected NoAuthException", + e.getCause() instanceof KeeperException.NoAuthException); + } + } + @Test public void testFencedState() throws Exception { TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
