Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 627469f31 -> 79983e00b
SENTRY-1814: Provide unit test for LeaderStatusMonitor (Alex Kolbasov, Reviewed by Vamsee Yarlagadda) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/79983e00 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/79983e00 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/79983e00 Branch: refs/heads/sentry-ha-redesign Commit: 79983e00bcda78a36cb855ccaa5d863250308abd Parents: 627469f Author: Alexander Kolbasov <[email protected]> Authored: Tue Jun 27 14:14:10 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Tue Jun 27 14:14:23 2017 -0700 ---------------------------------------------------------------------- .../db/service/persistent/HAContext.java | 21 +- .../service/thrift/LeaderStatusMonitor.java | 25 ++- .../db/service/thrift/TestActivator.java | 45 ----- .../service/thrift/TestLeaderStatusMonitor.java | 201 +++++++++++++++++++ 4 files changed, 243 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/79983e00/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/HAContext.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/HAContext.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/HAContext.java index e0f8a9e..0e5d606 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/HAContext.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/HAContext.java @@ -156,7 +156,9 @@ public final class HAContext implements AutoCloseable { public void run() { LOGGER.info("ShutdownHook closing curator framework"); try { - serverHAContext.close(); + if (serverHAContext != null) { + serverHAContext.close(); + } } catch (Throwable t) { LOGGER.error("Error stopping curator framework", t); } @@ -178,6 +180,23 @@ public final class HAContext implements AutoCloseable { return serverContext; } + /** + * Reset existing HA context. + * Should be only used by tests to provide different configurations. + */ + public static void resetHAContext() { + HAContext oldContext = serverHAContext; + if (oldContext != null) { + try { + oldContext.close(); + } catch (Exception e) { + LOGGER.error("Failed to close HACOntext", e); + } + } + serverHAContext = null; + } + + private void validateConf() { checkNotNull(zookeeperQuorum, "Zookeeper Quorum should not be null."); checkNotNull(namespace, "Zookeeper namespace should not be null."); http://git-wip-us.apache.org/repos/asf/sentry/blob/79983e00/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusMonitor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusMonitor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusMonitor.java index f78118c..360c5a5 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusMonitor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusMonitor.java @@ -16,6 +16,7 @@ */ package org.apache.sentry.service.thrift; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.curator.framework.CuratorFramework; @@ -96,7 +97,8 @@ final class LeaderStatusMonitor private final HAContext haContext; /** Unique string describing this instance */ - private final String incarnationId = generateIncarnationId(); + private final String defaultIncarnationId = generateIncarnationId(); + private String incarnationId; /** True when not using ZooKeeeper */ private final boolean isSingleNodeMode; @@ -126,17 +128,21 @@ final class LeaderStatusMonitor * which uses more properties. * @throws Exception */ - private LeaderStatusMonitor(Configuration conf) throws Exception { + + @VisibleForTesting + protected LeaderStatusMonitor(Configuration conf) throws Exception { // Only enable HA configuration if zookeeper is configured String zkServers = conf.get(SENTRY_HA_ZOOKEEPER_QUORUM, ""); if (zkServers.isEmpty()) { isSingleNodeMode = true; haContext = null; isLeader = true; + incarnationId = ""; LOG.info("Leader election protocol disabled, assuming single active server"); return; } isSingleNodeMode = false; + incarnationId = defaultIncarnationId; haContext = HAContext.getHAServerContext(conf); LOG.info("Created LeaderStatusMonitor(incarnationId=" + incarnationId + @@ -144,10 +150,23 @@ final class LeaderStatusMonitor } /** + * Tests may need to provide custm incarnation ID + * @param conf confguration + * @param incarnationId custom incarnation ID + * @throws Exception + */ + @VisibleForTesting + protected LeaderStatusMonitor(Configuration conf, String incarnationId) throws Exception { + this(conf); + this.incarnationId = incarnationId; + } + + /** * Second half of the constructor links this object with {@link HAContext} and * starts leader election protocol. */ - private void init() { + @VisibleForTesting + protected void init() { if (isSingleNodeMode) { return; } http://git-wip-us.apache.org/repos/asf/sentry/blob/79983e00/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestActivator.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestActivator.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestActivator.java deleted file mode 100644 index 5227c45..0000000 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestActivator.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.provider.db.service.thrift; - -import org.apache.sentry.service.thrift.SentryServiceIntegrationBase; -import org.junit.*; - -public class TestActivator extends SentryServiceIntegrationBase { - - @BeforeClass - public static void setup() throws Exception { - kerberos = false; - beforeSetup(); - setupConf(); - startSentryService(); - afterSetup(); - } - - @Override - @Before - public void before() throws Exception { - - } - - @Override - @After - public void after() { - - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/79983e00/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestLeaderStatusMonitor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestLeaderStatusMonitor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestLeaderStatusMonitor.java new file mode 100644 index 0000000..72d52de --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestLeaderStatusMonitor.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.service.thrift; + +import org.apache.curator.test.TestingServer; +import org.apache.hadoop.conf.Configuration; +import org.apache.sentry.provider.db.service.persistent.HAContext; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.lang.Thread.sleep; +import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Unit Tests for LeaderStatusMonitor. + * Use Curator TestingServer as Zookeeper Server. + */ +@SuppressWarnings("NestedTryStatement") +public class TestLeaderStatusMonitor { + private static final Logger LOGGER = LoggerFactory.getLogger(TestLeaderStatusMonitor.class); + + // Delay between retries + private static final int DELAY_MS = 500; + // Maximum number of tries before giving up while waiting for leader + private static final int NTRIES = 360; + // Number of times test is repeated + private static final int ITERATIONS = 10; + + /** + * Wait for some time (u to 500 seconds) until the monitor becomes active + * @param monitor HA monitor + * @return true if monitor is active, false otherwise + */ + @SuppressWarnings("squid:S2925") + private boolean isLeader(LeaderStatusMonitor monitor) { + for (int i = 0; i < NTRIES; i++) { + if (monitor.isLeader()) { + return true; + } + try { + sleep(DELAY_MS); + } catch (InterruptedException ignored) { + Thread.interrupted(); + } + } + return false; + } + + /** + * Simple test case - leader monitor without Zookeeper. + * Should always be the leader. + * @throws Exception + */ + @Test + public void testNoZk() throws Exception { + Configuration conf = new Configuration(); + LeaderStatusMonitor monitor = new LeaderStatusMonitor(conf); + assertTrue(monitor.isLeader()); + } + + /** + * Single server scenario. + * Should always be the leader. + * Should continue to be the leader after resigning the leadership. + * + * <p> + * <ol> + * <li>Start ZK Server</li> + * <li>Create monitor</li> + * <li>Monitor should become active</li> + * <li>Drop active status</li> + * <li>Monitor should become active again</li> + * </ol> + * @throws Exception + */ + @Test + @SuppressWarnings("squid:S2925") + public void testSingleServer() throws Exception { + try(TestingServer zkServer = new TestingServer()) { + zkServer.start(); + Configuration conf = new Configuration(); + conf.set(SENTRY_HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); + try(LeaderStatusMonitor monitor = new LeaderStatusMonitor(conf)) { + monitor.init(); + for (int i = 0; i < ITERATIONS; i++) { + assertTrue(isLeader(monitor)); + LOGGER.debug("testSingleServer(): deactivating leader"); + monitor.deactivate(); + sleep(2 * DELAY_MS); + assertTrue(isLeader(monitor)); + LOGGER.info("testSingleServer({}, leaderCount = {}", i, monitor.getLeaderCount()); + } + assertEquals(ITERATIONS + 1, monitor.getLeaderCount()); + } + } finally { + HAContext.resetHAContext(); + } + } + + /** + * Single server scenario with restarting ZK server + * <p> + * <ol> + * <li>Start ZK Server</li> + * <li>Create monitor</li> + * <li>at some point monitor should become active</li> + * <li>Restart ZK server</li> + * <li>at some point monitor should become active again</li> + * </ol> + * @throws Exception + */ + @Test + public void testSingleServerZkRestart() throws Exception { + try(TestingServer zkServer = new TestingServer()) { + zkServer.start(); + Configuration conf = new Configuration(); + conf.set(SENTRY_HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); + try(LeaderStatusMonitor monitor = new LeaderStatusMonitor(conf)) { + monitor.init(); + for (int i = 0; i < ITERATIONS; i++) { + assertTrue(isLeader(monitor)); + LOGGER.debug("testSingleServerZkRestart(): restarting Zk server"); + zkServer.restart(); + assertTrue(isLeader(monitor)); + LOGGER.info("testSingleServerZkRestart({}, leaderCount = {}", i, monitor.getLeaderCount()); + assertEquals(i + 2, monitor.getLeaderCount()); + } + } + } finally { + HAContext.resetHAContext(); + } + } + + /** + * Dual server scenario + * <p> + * <ol> + * <li>Start ZK Server</li> + * <li>Create monitor1 and monitor2</li> + * <li>at some point one of monitors should become active</li> + * <li>Drop active status on monitor 2</li> + * <li>Monitor1 should become active</li> + * <li>Drop active status on monitor1</li> + * <li>Monitor2 should become active</li> + * </ol> + * @throws Exception + */ + @Test + @SuppressWarnings("squid:S2925") + public void testTwoServers() throws Exception { + try(TestingServer zkServer = new TestingServer()) { + zkServer.start(); + Configuration conf = new Configuration(); + conf.set(SENTRY_HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); + try (LeaderStatusMonitor monitor1 = new LeaderStatusMonitor(conf, "1"); + LeaderStatusMonitor monitor2 = new LeaderStatusMonitor(conf, "2")) { + monitor1.init(); + monitor2.init(); + // Wait until one of monitors is active + for (int i = 0; i < NTRIES; i++) { + if (monitor1.isLeader() || monitor2.isLeader()) { + break; + } + try { + sleep(DELAY_MS); + } catch (InterruptedException ignored) { + Thread.interrupted(); + } + } + + for (int i = 0; i < ITERATIONS; i++) { + monitor2.deactivate(); + assertTrue(isLeader(monitor1)); + monitor1.deactivate(); + assertTrue(isLeader(monitor2)); + } + } + } finally { + HAContext.resetHAContext(); + } + } +}
