Repository: incubator-sentry Updated Branches: refs/heads/master 427705ff8 -> 36274c25c
SENTRY-648: Add e2e tests for Sentry HA (Prasad Mujumdar reviewed by Lenni Kuff) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/36274c25 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/36274c25 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/36274c25 Branch: refs/heads/master Commit: 36274c25c14370917981a53f63d8348a0c4a8bc4 Parents: 427705f Author: Prasad Mujumdar <pras...@apache.org> Authored: Fri Feb 20 00:38:32 2015 -0800 Committer: Prasad Mujumdar <pras...@apache.org> Committed: Fri Feb 20 00:38:32 2015 -0800 ---------------------------------------------------------------------- .../db/service/persistent/ServiceManager.java | 24 ++- .../thrift/HAClientInvocationHandler.java | 50 +++--- .../sentry/service/thrift/SentryService.java | 24 ++- .../sentry/service/thrift/ServiceConstants.java | 2 +- .../sentry/tests/e2e/ha/TestHaEnd2End.java | 171 +++++++++++++++++++ .../AbstractTestWithStaticConfiguration.java | 62 ++++--- .../tests/e2e/minisentry/InternalSentrySrv.java | 164 ++++++++++++++++++ .../sentry/tests/e2e/minisentry/SentrySrv.java | 79 +++++++++ .../tests/e2e/minisentry/SentrySrvFactory.java | 45 +++++ 9 files changed, 564 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java index 2274f00..a970b92 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java @@ -29,11 +29,20 @@ import org.apache.curator.x.discovery.ServiceInstance; import org.apache.curator.x.discovery.ServiceProvider; import org.apache.curator.x.discovery.details.InstanceSerializer; import org.apache.hadoop.net.NetUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/*** + * ServerManager handles registration of the Sentry service for Curator service + * discovery. Each server registers with ZK and add its host:port details which + * is used by the clients to discover available servers + */ public class ServiceManager { - + private static final Logger LOGGER = LoggerFactory + .getLogger(ServiceManager.class); private HAContext haContext; private ServiceProvider<Void> serviceProvider; + private ServiceDiscovery<Void> serviceDiscovery; public ServiceManager(HAContext haContext) throws IOException { this.haContext = haContext; @@ -47,7 +56,7 @@ public class ServiceManager { curatorFramework.start(); } InstanceSerializer<Void> instanceSerializer = new FixedJsonInstanceSerializer<Void>(Void.class); - ServiceDiscovery<Void> serviceDiscovery = ServiceDiscoveryBuilder.<Void>builder(Void.class) + serviceDiscovery = ServiceDiscoveryBuilder.<Void>builder(Void.class) .basePath(HAContext.SENTRY_SERVICE_REGISTER_NAMESPACE) .serializer(instanceSerializer) .client(curatorFramework) @@ -80,4 +89,15 @@ public class ServiceManager { public static InetSocketAddress convertServiceInstance(ServiceInstance<?> service) { return NetUtils.createSocketAddr(service.getAddress(),service.getPort()); } + + public void close() { + try { + serviceProvider.close(); + serviceDiscovery.close(); + haContext.getCuratorFramework().close(); + LOGGER.debug("Closed ZK resources"); + } catch (IOException e) { + LOGGER.warn("Error closing the service manager", e); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java index c6e265f..1dea938 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java @@ -25,7 +25,6 @@ import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; - import org.apache.curator.x.discovery.ServiceInstance; import org.apache.sentry.SentryUserException; import org.apache.sentry.provider.db.service.persistent.HAContext; @@ -43,7 +42,7 @@ public class HAClientInvocationHandler implements InvocationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(HAClientInvocationHandler.class); private final Configuration conf; - private final ServiceManager manager; + private ServiceManager manager; private ServiceInstance<Void> currentServiceInstance; private SentryPolicyServiceClient client = null; @@ -51,7 +50,6 @@ public class HAClientInvocationHandler implements InvocationHandler { public HAClientInvocationHandler(Configuration conf) throws Exception { this.conf = conf; - manager = new ServiceManager(new HAContext(conf)); checkClientConf(); renewSentryClient(); } @@ -87,25 +85,35 @@ public class HAClientInvocationHandler implements InvocationHandler { } private void renewSentryClient() throws IOException { - while (true) { - if (client != null) { - client.close(); - } - currentServiceInstance = manager.getServiceInstance(); - if (currentServiceInstance == null) { - throw new IOException("No avaiable node."); - } - InetSocketAddress serverAddress = - ServiceManager.convertServiceInstance(currentServiceInstance); - conf.set(ServiceConstants.ClientConfig.SERVER_RPC_ADDRESS, serverAddress.getHostName()); - conf.setInt(ServiceConstants.ClientConfig.SERVER_RPC_PORT, serverAddress.getPort()); - try { - client = new SentryPolicyServiceClientDefaultImpl(conf); - break; - } catch (IOException e) { - manager.reportError(currentServiceInstance); - LOGGER.info("Transport exception while opening transport:", e, e.getMessage()); + try { + manager = new ServiceManager(new HAContext(conf)); + } catch (Exception e1) { + throw new IOException("Failed to extract Sentry node info from zookeeper", e1); + } + + try { + while (true) { + if (client != null) { + client.close(); + } + currentServiceInstance = manager.getServiceInstance(); + if (currentServiceInstance == null) { + throw new IOException("No avaiable node."); + } + InetSocketAddress serverAddress = + ServiceManager.convertServiceInstance(currentServiceInstance); + conf.set(ServiceConstants.ClientConfig.SERVER_RPC_ADDRESS, serverAddress.getHostName()); + conf.setInt(ServiceConstants.ClientConfig.SERVER_RPC_PORT, serverAddress.getPort()); + try { + client = new SentryPolicyServiceClientDefaultImpl(conf); + break; + } catch (IOException e) { + manager.reportError(currentServiceInstance); + LOGGER.info("Transport exception while opening transport:", e, e.getMessage()); + } } + } finally { + manager.close(); } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java index 02788aa..0b4b39a 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java @@ -85,7 +85,7 @@ public class SentryService implements Callable { private final String[] principalParts; private final String keytab; private final ExecutorService serviceExecutor; - private Future future; + private Future serviceStatus; private TServer thriftServer; private Status status; private int webServerPort; @@ -239,6 +239,7 @@ public class SentryService implements Callable { private void stopSentryWebServer() throws Exception{ if( sentryWebServer != null) { sentryWebServer.stop(); + sentryWebServer = null; } } @@ -257,7 +258,7 @@ public class SentryService implements Callable { } LOGGER.info("Attempting to start..."); status = Status.STARTED; - future = serviceExecutor.submit(this); + serviceStatus = serviceExecutor.submit(this); } public synchronized void stop() throws Exception{ @@ -295,6 +296,16 @@ public class SentryService implements Callable { LOGGER.info("Stopped..."); } + // wait for the service thread to finish execution + public synchronized void waitForShutDown() { + LOGGER.info("Waiting on future.get()"); + try { + serviceStatus.get(); + } catch (Exception e) { + LOGGER.debug("Error during the shutdown", e); + } + } + private MultiException addMultiException(MultiException exception, Exception e) { if(exception == null){ exception = new MultiException(); @@ -355,9 +366,9 @@ public class SentryService implements Callable { File configFile = null; if (configFileName == null || commandLine.hasOption("h") || commandLine.hasOption("help")) { // print usage - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("sentry --command service", options); - System.exit(-1); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("sentry --command service", options); + System.exit(-1); } else if(!((configFile = new File(configFileName)).isFile() && configFile.canRead())) { throw new IllegalArgumentException("Cannot read configuration file " + configFile); } @@ -378,8 +389,7 @@ public class SentryService implements Callable { // Let's wait on the service to stop try { - LOGGER.info("Waiting on future.get()"); - server.future.get(); + server.waitForShutDown(); } finally { server.serviceExecutor.shutdown(); } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java index ddc5930..77e1f60 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java @@ -121,7 +121,7 @@ public class ServiceConstants { public static final String SENTRY_HA_ZOOKEEPER_SLEEP_BETWEEN_RETRIES_MS = SENTRY_HA_ZK_PROPERTY_PREFIX + "session.sleep.between.retries.ms"; public static final int SENTRY_HA_ZOOKEEPER_SLEEP_BETWEEN_RETRIES_MS_DEFAULT = 100; public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE = SENTRY_HA_ZK_PROPERTY_PREFIX + "namespace"; - public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE_DEFAULT = "/sentry"; + public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE_DEFAULT = "sentry"; public static final ImmutableMap<String, String> SENTRY_STORE_DEFAULTS = ImmutableMap.<String, String>builder() http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/ha/TestHaEnd2End.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/ha/TestHaEnd2End.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/ha/TestHaEnd2End.java new file mode 100644 index 0000000..78894d1 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/ha/TestHaEnd2End.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.ha; + +import java.io.File; +import java.io.FileOutputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import org.apache.sentry.provider.db.SentryAccessDeniedException; +import org.apache.sentry.provider.file.PolicyFile; +import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; +import org.apache.sentry.tests.e2e.hive.StaticUserGroup; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.io.Resources; + +/** + * End2End tests with Sentry service HA enabled. + */ +public class TestHaEnd2End extends AbstractTestWithStaticConfiguration { + + private final String SINGLE_TYPE_DATA_FILE_NAME = "kv1.dat"; + private File dataFile; + private PolicyFile policyFile; + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception { + useSentryService = true; + enableSentryHA = true; + AbstractTestWithStaticConfiguration.setupTestStaticConfiguration(); + } + + @Override + @Before + public void setup() throws Exception { + super.setupAdmin(); + super.setup(); + dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME); + FileOutputStream to = new FileOutputStream(dataFile); + Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); + to.close(); + policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); + } + + /** + * Basic test with two Sentry service running. + * @throws Exception + */ + @Test + public void testBasic() throws Exception { + Connection connection = context.createConnection(ADMIN1); + Statement statement = context.createStatement(connection); + statement.execute("CREATE TABLE t1 (c1 string)"); + statement.execute("CREATE ROLE user_role"); + statement.execute("GRANT SELECT ON TABLE t1 TO ROLE user_role"); + statement.execute("GRANT ROLE user_role TO GROUP " + USERGROUP1); + statement.close(); + connection.close(); + connection = context.createConnection(USER1_1); + statement = context.createStatement(connection); + context.assertSentryException(statement, "CREATE ROLE r2", + SentryAccessDeniedException.class.getSimpleName()); + // test default of ALL + statement.execute("SELECT * FROM t1"); + // test a specific role + statement.execute("SET ROLE user_role"); + statement.execute("SELECT * FROM t1"); + + // test ALL + statement.execute("SET ROLE ALL"); + statement.execute("SELECT * FROM t1"); + statement.close(); + connection.close(); + + // cleanup + connection = context.createConnection(ADMIN1); + statement = context.createStatement(connection); + statement.execute("DROP ROLE user_role"); + statement.close(); + connection.close(); + } + + /** + * Test service failover. Run Sentry operations with shutting down one or more + * of the services. + * @throws Exception + */ + @Test + public void testFailover() throws Exception { + String roleName1 = "test_role_1"; + String roleName2 = "test_role_2"; + String roleName3 = "test_role_3"; + + dropDb(ADMIN1, DB1); + createDb(ADMIN1, DB1); + createTable(ADMIN1, DB1, dataFile, TBL1); + + Connection adminCon = context.createConnection(ADMIN1); + Statement adminStmt = context.createStatement(adminCon); + // access the new databases + adminStmt.execute("use " + DB1); + + // stop server1 and verify sentry continues to work + getSentrySrv().stop(0); + adminStmt.execute("CREATE ROLE " + roleName1); + verifyRoleExists(adminStmt, roleName1); + + // restart server1 and stop server2 + getSentrySrv().start(0); + getSentrySrv().stop(1); + adminStmt.execute("CREATE ROLE " + roleName2); + verifyRoleExists(adminStmt, roleName2); + + // stop both servers and verify it fails + getSentrySrv().stop(0); + getSentrySrv().stop(1); + context.assertAuthzException(adminStmt, "CREATE ROLE " + roleName3); + + getSentrySrv().start(0); + getSentrySrv().start(1); + adminStmt.execute("CREATE ROLE " + roleName3); + verifyRoleExists(adminStmt, roleName3); + + // cleanup + + dropDb(ADMIN1, DB1); + adminStmt.execute("DROP ROLE " + roleName1); + adminStmt.execute("DROP ROLE " + roleName2); + adminStmt.execute("DROP ROLE " + roleName3); + adminStmt.close(); + adminCon.close(); + + } + + private void verifyRoleExists(Statement statement, String roleName) + throws Exception { + ResultSet resultSet = null; + try { + resultSet = statement.executeQuery("SHOW ROLES "); + while (resultSet.next()) { + if (roleName.equalsIgnoreCase(resultSet.getString(1))) { + return; + } + } + throw new Exception("Role " + roleName + " does not exist"); + } finally { + if (resultSet != null) { + resultSet.close(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java index c68873e..3a8a6ef 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java @@ -49,15 +49,16 @@ import org.apache.sentry.policy.db.DBModelAuthorizables; import org.apache.sentry.provider.db.SimpleDBProviderBackend; import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; import org.apache.sentry.provider.file.PolicyFile; -import org.apache.sentry.service.thrift.SentryService; import org.apache.sentry.service.thrift.SentryServiceClientFactory; -import org.apache.sentry.service.thrift.SentryServiceFactory; import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.apache.sentry.tests.e2e.hive.fs.DFS; import org.apache.sentry.tests.e2e.hive.fs.DFSFactory; import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServer; import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; +import org.apache.sentry.tests.e2e.minisentry.SentrySrvFactory; +import org.apache.sentry.tests.e2e.minisentry.SentrySrvFactory.SentrySrvType; +import org.apache.sentry.tests.e2e.minisentry.SentrySrv; import org.apache.tools.ant.util.StringUtils; import org.junit.After; import org.junit.AfterClass; @@ -108,6 +109,7 @@ public abstract class AbstractTestWithStaticConfiguration { protected static final String SERVER_HOST = "localhost"; private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry"; protected static final String EXTERNAL_HIVE_LIB = "sentry.e2etest.hive.lib"; + private static final String ENABLE_SENTRY_HA = "sentry.e2etest.enable.service.ha"; protected static boolean policyOnHdfs = false; protected static boolean useSentryService = false; @@ -127,8 +129,9 @@ public abstract class AbstractTestWithStaticConfiguration { protected static HiveServerFactory.HiveServer2Type hiveServer2Type; protected static DFS dfs; protected static Map<String, String> properties; - protected static SentryService sentryServer; + protected static SentrySrv sentryServer; protected static Configuration sentryConf; + protected static boolean enableSentryHA = false; protected static Context context; protected final String semanticException = "SemanticException No valid privileges"; @@ -209,7 +212,7 @@ public abstract class AbstractTestWithStaticConfiguration { PolicyFile policyFile = PolicyFile.setAdminOnServer1(ADMIN1) .setUserGroupMapping(StaticUserGroup.getStaticMapping()); policyFile.write(policyFileLocation); - + String policyURI; if (policyOnHdfs) { String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString(); @@ -220,8 +223,11 @@ public abstract class AbstractTestWithStaticConfiguration { } else { policyURI = policyFileLocation.getPath(); } - + boolean startSentry = new Boolean(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false")); + if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) { + enableSentryHA = true; + } if (useSentryService && (!startSentry)) { setupSentryService(); } @@ -240,8 +246,8 @@ public abstract class AbstractTestWithStaticConfiguration { } public static HiveServer create(Map<String, String> properties, - File baseDir, File confDir, File logDir, String policyFile, - FileSystem fileSystem) throws Exception { + File baseDir, File confDir, File logDir, String policyFile, + FileSystem fileSystem) throws Exception { String type = properties.get(HiveServerFactory.HIVESERVER2_TYPE); if(type == null) { type = System.getProperty(HiveServerFactory.HIVESERVER2_TYPE); @@ -291,7 +297,7 @@ public abstract class AbstractTestWithStaticConfiguration { .entrySet()) { for (String roleNames : groupEntry.getValue()) { for (String roleName : roleNames.split(",")) { - statement.execute("GRANT ROLE " + roleName + " TO GROUP " + groupEntry.getKey()); + statement.execute("GRANT ROLE " + roleName + " TO GROUP " + groupEntry.getKey()); } } } @@ -359,7 +365,7 @@ public abstract class AbstractTestWithStaticConfiguration { properties.put(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname, SentryHiveAuthorizationTaskFactoryImpl.class.getName()); properties - .put(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS.varname, "2"); + .put(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS.varname, "2"); properties.put(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE); properties.put(ServerConfig.ADMIN_GROUPS, ADMINGROUP); properties.put(ServerConfig.RPC_ADDRESS, SERVER_HOST); @@ -368,22 +374,30 @@ public abstract class AbstractTestWithStaticConfiguration { properties.put(ServerConfig.SENTRY_STORE_JDBC_URL, "jdbc:derby:;databaseName=" + baseDir.getPath() - + "/sentrystore_db;create=true"); + + "/sentrystore_db;create=true"); properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING); properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath()); properties.put(ServerConfig.RPC_MIN_THREADS, "3"); for (Map.Entry<String, String> entry : properties.entrySet()) { sentryConf.set(entry.getKey(), entry.getValue()); } - sentryServer = new SentryServiceFactory().create(sentryConf); - properties.put(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.getAddress() + sentryServer = SentrySrvFactory.create( + SentrySrvType.INTERNAL_SERVER, sentryConf, enableSentryHA ? 2 : 1); + properties.put(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.get(0) + .getAddress() .getHostName()); - sentryConf.set(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.getAddress() + sentryConf.set(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.get(0) + .getAddress() .getHostName()); properties.put(ClientConfig.SERVER_RPC_PORT, - String.valueOf(sentryServer.getAddress().getPort())); + String.valueOf(sentryServer.get(0).getAddress().getPort())); sentryConf.set(ClientConfig.SERVER_RPC_PORT, - String.valueOf(sentryServer.getAddress().getPort())); + String.valueOf(sentryServer.get(0).getAddress().getPort())); + if (enableSentryHA) { + properties.put(ClientConfig.SERVER_HA_ENABLED, "true"); + properties.put(ClientConfig.SENTRY_HA_ZOOKEEPER_QUORUM, + sentryServer.getZKQuorum()); + } startSentryService(); if (setMetastoreListener) { properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, @@ -393,21 +407,14 @@ public abstract class AbstractTestWithStaticConfiguration { } private static void startSentryService() throws Exception { - sentryServer.start(); - final long start = System.currentTimeMillis(); - while (!sentryServer.isRunning()) { - Thread.sleep(1000); - if (System.currentTimeMillis() - start > 60000L) { - throw new TimeoutException("Server did not start after 60 seconds"); - } - } + sentryServer.startAll(); } public static SentryPolicyServiceClient getSentryClient() throws Exception { if (sentryServer == null) { throw new IllegalAccessException("Sentry service not initialized"); } - return SentryServiceClientFactory.create(sentryServer.getConf()); + return SentryServiceClientFactory.create(sentryServer.get(0).getConf()); } @Before @@ -475,8 +482,7 @@ public abstract class AbstractTestWithStaticConfiguration { } if (sentryServer != null) { - sentryServer.stop(); - sentryServer = null; + sentryServer.close(); sentryServer = null; } @@ -497,4 +503,8 @@ public abstract class AbstractTestWithStaticConfiguration { context.close(); } } + + public static SentrySrv getSentrySrv() { + return sentryServer; + } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java new file mode 100644 index 0000000..68bc9ee --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.minisentry; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeoutException; + +import org.apache.curator.test.TestingServer; +import org.apache.hadoop.conf.Configuration; +import org.apache.sentry.service.thrift.SentryService; +import org.apache.sentry.service.thrift.SentryServiceFactory; +import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig; +import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class InternalSentrySrv implements SentrySrv { + + private List<SentryService> sentryServers = Lists.newArrayList(); + private static TestingServer zkServer; // created only if in case of HA + private static final Logger LOGGER = LoggerFactory + .getLogger(InternalSentrySrv.class); + private boolean isActive = false; + + public InternalSentrySrv(Configuration sentryConf, int numServers) + throws Exception { + // Enable HA when numServers is more that 1, start Curator TestingServer + if (numServers > 1) { + zkServer = new TestingServer(); + zkServer.start(); + sentryConf.setBoolean(ServerConfig.SENTRY_HA_ENABLED, true); + sentryConf.set(ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM, + zkServer.getConnectString()); + } else if (numServers <= 0) { + throw new IllegalArgumentException("Invalid number of Servers: " + + numServers + " ,must be > 0"); + } + for (int count = 0; count < numServers; count++) { + Configuration servConf = new Configuration(sentryConf); + SentryService sentryServer = new SentryServiceFactory().create(servConf); + servConf.set(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.getAddress() + .getHostName()); + servConf.setInt(ClientConfig.SERVER_RPC_PORT, sentryServer.getAddress() + .getPort()); + sentryServers.add(sentryServer); + } + isActive = true; + } + + @Override + public void startAll() throws Exception { + if (!isActive) { + throw new IllegalStateException("SentrySrv is no longer active"); + } + for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++) { + start(sentryServerNum); + } + } + + @Override + public void start(int serverNum) throws Exception { + if (!isActive) { + throw new IllegalStateException("SentrySrv is no longer active"); + } + SentryService sentryServer = sentryServers.get(serverNum); + sentryServer.start(); + + // wait for startup + final long start = System.currentTimeMillis(); + while (!sentryServer.isRunning()) { + Thread.sleep(1000); + if (System.currentTimeMillis() - start > 60000L) { + throw new TimeoutException("Server did not start after 60 seconds"); + } + } + } + + @Override + public void stopAll() throws Exception { + boolean cleanStop = true; + if (!isActive) { + throw new IllegalStateException("SentrySrv is no longer active"); + } + for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++) { + try { + stop(sentryServerNum); + } catch (Exception e) { + LOGGER.error("Sentry Server " + sentryServerNum + " failed to stop"); + cleanStop = false; + } + } + if (!cleanStop) { + throw new IllegalStateException( + "At least one of the servers failed to stop cleanly"); + } + } + + @Override + public void stop(int serverNum) throws Exception { + if (!isActive) { + throw new IllegalStateException("SentrySrv is no longer active"); + } + SentryService sentryServer = sentryServers.get(serverNum); + sentryServer.stop(); + sentryServer.waitForShutDown(); + } + + @Override + public void close() { + for (SentryService sentryServer : sentryServers) { + try { + sentryServer.stop(); + } catch (Exception e) { + LOGGER.error("Error stoping Sentry service ", e); + } + } + if (zkServer != null) { + try { + zkServer.stop(); + } catch (IOException e) { + LOGGER.warn("Error stoping ZK service ", e); + } + } + sentryServers.clear(); + isActive = false; + } + + @Override + public SentryService get(int serverNum) { + return sentryServers.get(serverNum); + } + + @Override + public String getZKQuorum() throws Exception { + if (zkServer == null) { + throw new IOException("Sentry HA is not enabled"); + } + return zkServer.getConnectString(); + } + + @Override + public boolean isHaEnabled() { + return zkServer != null; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java new file mode 100644 index 0000000..e9ae5fa --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.minisentry; + +import org.apache.sentry.service.thrift.SentryService; + +public interface SentrySrv { + + /** + * Start all the sentry services + * @throws Exception + */ + public void startAll() throws Exception; + + /** + * Start the given server + * @param serverNum + * - Server number (0 to N-1) + * @throws Exception + */ + public void start(int serverNum) throws Exception ; + + /** + * Stop all the Sentry servers + * @throws Exception + */ + public void stopAll() throws Exception; + + /** + * Stop the specified Sentry server + * @param serverNum + * - Server number (0 to N-1) + * @throws Exception + */ + public void stop(int serverNum) throws Exception ; + + /** + * Get the underlying Sentry service object + * @param serverNum + * - Server number (0 to N-1) + * @return + */ + public SentryService get(int serverNum); + + /** + * Get the ZK connection string + * @return + * @throws Exception + * - If HA is not enabled + */ + public String getZKQuorum() throws Exception; + + /** + * Stop all the nodes and ZK if started. The SentrySrv can't be reused once + * closed. + */ + public void close(); + + /** + * Check if the sentry server is created with HA enabled. + * @return True - HA is enabled False - HA is not enabled + */ + public boolean isHaEnabled(); +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/36274c25/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java new file mode 100644 index 0000000..9381e88 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.minisentry; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +public class SentrySrvFactory { + + public static enum SentrySrvType { + INTERNAL_SERVER, EXTERNAL_SERVER + } + + public static SentrySrv create(SentrySrvType srvType, + Configuration sentryConf) + throws Exception { + return create(srvType, sentryConf, 1); + } + + public static SentrySrv create(SentrySrvType srvType, + Configuration sentryConf, + int numServers) throws Exception { + if (!srvType.equals(SentrySrvType.INTERNAL_SERVER)) { + throw new IOException("Server type " + srvType.name() + + " is not supported"); + } + return new InternalSentrySrv(sentryConf, numServers); + } +}
