http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java deleted file mode 100644 index a79ce5f..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.service.thrift; - -import java.util.HashMap; -import java.util.Map; - -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.Configuration; - -/** - * Creates a programmatic version of a jaas.conf file. This can be used instead of writing a jaas.conf file and setting - * the system property, "java.security.auth.login.config", to point to that file. It is meant to be used for connecting to - * ZooKeeper. - * <p> - * example usage: - * JaasConfiguration.addEntry("Client", principal, keytabFile); - * javax.security.auth.login.Configuration.setConfiguration(JaasConfiguration.getInstance()); - */ -public final class JaasConfiguration extends Configuration { - private static Map<String, AppConfigurationEntry> entries = new HashMap<String, AppConfigurationEntry>(); - private static JaasConfiguration me = null; - private static final String krb5LoginModuleName; - - static { - if (System.getProperty("java.vendor").contains("IBM")) { - krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule"; - } - else { - krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; - } - } - - private JaasConfiguration() { - // don't need to do anything here but we want to make it private - } - - /** - * Return the singleton. You'd typically use it only to do this: - * <p> - * javax.security.auth.login.Configuration.setConfiguration(JaasConfiguration.getInstance()); - * - * @return - */ - public static Configuration getInstance() { - if (me == null) { - me = new JaasConfiguration(); - } - return me; - } - - /** - * Add an entry to the jaas configuration with the passed in name, principal, and keytab. The other necessary options will be - * set for you. - * - * @param name The name of the entry (e.g. "Client") - * @param principal The principal of the user - * @param keytab The location of the keytab - */ - public static void addEntryForKeytab(String name, String principal, String keytab) { - Map<String, String> options = new HashMap<String, String>(); - options.put("keyTab", keytab); - options.put("principal", principal); - options.put("useKeyTab", "true"); - options.put("storeKey", "true"); - options.put("useTicketCache", "false"); - AppConfigurationEntry entry = new AppConfigurationEntry(krb5LoginModuleName, - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options); - entries.put(name, entry); - } - - /** - * Add an entry to the jaas configuration with the passed in name. The other - * necessary options will be set for you. - * - * @param name The name of the entry (e.g. "Client") - */ - public static void addEntryForTicketCache(String sectionName) { - Map<String, String> options = new HashMap<String, String>(); - options.put("useKeyTab", "false"); - options.put("storeKey", "false"); - options.put("useTicketCache", "true"); - AppConfigurationEntry entry = new AppConfigurationEntry(krb5LoginModuleName, - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options); - entries.put(sectionName, entry); - } - - /** - * Removes the specified entry. - * - * @param name The name of the entry to remove - */ - public static void removeEntry(String name) { - entries.remove(name); - } - - /** - * Clears all entries. - */ - public static void clearEntries() { - entries.clear(); - } - - /** - * Returns the entries map. - * - * @return the entries map - */ - public static Map<String, AppConfigurationEntry> getEntries() { - return entries; - } - - @Override - public AppConfigurationEntry[] getAppConfigurationEntry(String name) { - return new AppConfigurationEntry[]{entries.get(name)}; - } -} -
http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java deleted file mode 100644 index 41e4fe4..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.service.thrift; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -import javax.security.auth.login.AppConfigurationEntry; - -public class KerberosConfiguration extends javax.security.auth.login.Configuration { - private String principal; - private String keytab; - private boolean isInitiator; - private static final boolean IBM_JAVA = System.getProperty("java.vendor").contains("IBM"); - - private KerberosConfiguration(String principal, File keytab, - boolean client) { - this.principal = principal; - this.keytab = keytab.getAbsolutePath(); - this.isInitiator = client; - } - - public static javax.security.auth.login.Configuration createClientConfig(String principal, - File keytab) { - return new KerberosConfiguration(principal, keytab, true); - } - - public static javax.security.auth.login.Configuration createServerConfig(String principal, - File keytab) { - return new KerberosConfiguration(principal, keytab, false); - } - - private static String getKrb5LoginModuleName() { - return (IBM_JAVA ? "com.ibm.security.auth.module.Krb5LoginModule" - : "com.sun.security.auth.module.Krb5LoginModule"); - } - - @Override - public AppConfigurationEntry[] getAppConfigurationEntry(String name) { - Map<String, String> options = new HashMap<String, String>(); - - if (IBM_JAVA) { - // IBM JAVA's UseKeytab covers both keyTab and useKeyTab options - options.put("useKeytab",keytab.startsWith("file://") ? keytab : "file://" + keytab); - - options.put("principal", principal); - options.put("refreshKrb5Config", "true"); - - // Both "initiator" and "acceptor" - options.put("credsType", "both"); - } else { - options.put("keyTab", keytab); - options.put("principal", principal); - options.put("useKeyTab", "true"); - options.put("storeKey", "true"); - options.put("doNotPrompt", "true"); - options.put("useTicketCache", "true"); - options.put("renewTGT", "true"); - options.put("refreshKrb5Config", "true"); - options.put("isInitiator", Boolean.toString(isInitiator)); - } - - String ticketCache = System.getenv("KRB5CCNAME"); - if (IBM_JAVA) { - // If cache is specified via env variable, it takes priority - if (ticketCache != null) { - // IBM JAVA only respects system property so copy ticket cache to system property - // The first value searched when "useDefaultCcache" is true. - System.setProperty("KRB5CCNAME", ticketCache); - } else { - ticketCache = System.getProperty("KRB5CCNAME"); - } - - if (ticketCache != null) { - options.put("useDefaultCcache", "true"); - options.put("renewTGT", "true"); - } - } else { - if (ticketCache != null) { - options.put("ticketCache", ticketCache); - } - } - options.put("debug", "true"); - - return new AppConfigurationEntry[]{ - new AppConfigurationEntry(getKrb5LoginModuleName(), - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - options)}; - } -} - http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java deleted file mode 100644 index 2a48c63..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.service.thrift; - -import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.provider.db.service.persistent.SentryStore; -import org.apache.thrift.TMultiplexedProcessor; - -public abstract class ProcessorFactory { - protected final Configuration conf; - - public ProcessorFactory(Configuration conf) { - this.conf = conf; - } - - /** - * Register a Thrift processor with SentryStore. - * @param processor a thrift processor. - * @param sentryStore a {@link SentryStore} - * @return true if success. - * @throws Exception - */ - public abstract boolean register(TMultiplexedProcessor processor, - SentryStore sentryStore) throws Exception; -} http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java deleted file mode 100644 index b9a0563..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - <p> - http://www.apache.org/licenses/LICENSE-2.0 - <p> - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.apache.sentry.service.thrift; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Timer; -import com.codahale.metrics.Timer.Context; -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; -import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; -import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageDeserializer; -import org.apache.sentry.provider.db.service.persistent.PathsImage; -import org.apache.sentry.provider.db.service.persistent.SentryStore; -import org.apache.sentry.api.service.thrift.SentryMetrics; -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; - -import static com.codahale.metrics.MetricRegistry.name; -import static java.util.Collections.emptyMap; - -/** - * Wrapper class for <Code>HiveMetaStoreClient</Code> - * - * <p>Abstracts communication with HMS and exposes APi's to connect/disconnect to HMS and to - * request HMS snapshots and also for new notifications. - */ -public class SentryHMSClient implements AutoCloseable { - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryHMSClient.class); - private static final String NOT_CONNECTED_MSG = "Client is not connected to HMS"; - - private final Configuration conf; - private HiveMetaStoreClient client = null; - private HiveConnectionFactory hiveConnectionFactory; - - private static final String SNAPSHOT = "snapshot"; - /** Measures time to get full snapshot. */ - private final Timer updateTimer = SentryMetrics.getInstance() - .getTimer(name(FullUpdateInitializer.class, SNAPSHOT)); - /** Number of times update failed. */ - private final Counter failedSnapshotsCount = SentryMetrics.getInstance() - .getCounter(name(FullUpdateInitializer.class, "failed")); - - public SentryHMSClient(Configuration conf, HiveConnectionFactory hiveConnectionFactory) { - this.conf = conf; - this.hiveConnectionFactory = hiveConnectionFactory; - } - - /** - * Used only for testing purposes. - *x - * @param client HiveMetaStoreClient to be initialized - */ - @VisibleForTesting - void setClient(HiveMetaStoreClient client) { - this.client = client; - } - - /** - * Used to know if the client is connected to HMS - * - * @return true if the client is connected to HMS false, if client is not connected. - */ - boolean isConnected() { - return client != null; - } - - /** - * Connects to HMS by creating HiveMetaStoreClient. - * - * @throws IOException if could not establish connection - * @throws InterruptedException if connection was interrupted - * @throws MetaException if other errors happened - */ - public void connect() - throws IOException, InterruptedException, MetaException { - if (client != null) { - return; - } - client = hiveConnectionFactory.connect().getClient(); - } - - /** - * Disconnects the HMS client. - */ - public void disconnect() throws Exception { - try { - if (client != null) { - LOGGER.info("Closing the HMS client connection"); - client.close(); - } - } catch (Exception e) { - LOGGER.error("failed to close Hive Connection Factory", e); - } finally { - client = null; - } - } - - /** - * Closes the HMS client. - * - * <p>This is similar to disconnect. As this class implements AutoClosable, close should be - * implemented. - */ - public void close() throws Exception { - disconnect(); - } - - /** - * Creates HMS full snapshot. - * - * @return Full path snapshot and the last notification id on success - */ - public PathsImage getFullSnapshot() { - if (client == null) { - LOGGER.error(NOT_CONNECTED_MSG); - return new PathsImage(Collections.<String, Collection<String>>emptyMap(), - SentryStore.EMPTY_NOTIFICATION_ID, SentryStore.EMPTY_PATHS_SNAPSHOT_ID); - } - - try { - CurrentNotificationEventId eventIdBefore = client.getCurrentNotificationEventId(); - Map<String, Collection<String>> pathsFullSnapshot = fetchFullUpdate(); - if (pathsFullSnapshot.isEmpty()) { - LOGGER.info("Received empty paths when getting full snapshot. NotificationID Before Snapshot: {}", eventIdBefore.getEventId()); - return new PathsImage(pathsFullSnapshot, SentryStore.EMPTY_NOTIFICATION_ID, - SentryStore.EMPTY_PATHS_SNAPSHOT_ID); - } - - CurrentNotificationEventId eventIdAfter = client.getCurrentNotificationEventId(); - LOGGER.info("NotificationID, Before Snapshot: {}, After Snapshot {}", - eventIdBefore.getEventId(), eventIdAfter.getEventId()); - - if (eventIdAfter.equals(eventIdBefore)) { - LOGGER.info("Successfully fetched hive full snapshot, Current NotificationID: {}.", - eventIdAfter); - // As eventIDAfter is the last event that was processed, eventIDAfter is used to update - // lastProcessedNotificationID instead of getting it from persistent store. - return new PathsImage(pathsFullSnapshot, eventIdAfter.getEventId(), - SentryStore.EMPTY_PATHS_SNAPSHOT_ID); - } - - LOGGER.info("Reconciling full snapshot - applying {} changes", - eventIdAfter.getEventId() - eventIdBefore.getEventId()); - - // While we were taking snapshot, HMS made some changes, so now we need to apply all - // extra events to the snapshot - long currentEventId = eventIdBefore.getEventId(); - MessageDeserializer deserializer = new SentryJSONMessageDeserializer(); - - while (currentEventId < eventIdAfter.getEventId()) { - NotificationEventResponse response = - client.getNextNotification(currentEventId, Integer.MAX_VALUE, null); - if (response == null || !response.isSetEvents() || response.getEvents().isEmpty()) { - LOGGER.error("Snapshot discarded, updates to HMS data while shapshot is being taken." - + "ID Before: {}. ID After: {}", eventIdBefore.getEventId(), eventIdAfter.getEventId()); - return new PathsImage(Collections.<String, Collection<String>>emptyMap(), - SentryStore.EMPTY_NOTIFICATION_ID, SentryStore.EMPTY_PATHS_SNAPSHOT_ID); - } - - for (NotificationEvent event : response.getEvents()) { - LOGGER.info("Received event = {} currentEventId = {}, eventIdAfter = {}", event.getEventId(), currentEventId, eventIdAfter); - if (event.getEventId() <= eventIdBefore.getEventId()) { - LOGGER.error("Received stray event with eventId {} which is less then {}", - event.getEventId(), eventIdBefore); - continue; - } - if (event.getEventId() > eventIdAfter.getEventId()) { - // Enough events processed - LOGGER.debug("Received eventId = {} is greater than eventIdAfter = {}", event.getEventId(), eventIdAfter); - break; - } - try { - FullUpdateModifier.applyEvent(pathsFullSnapshot, event, deserializer); - } catch (Exception e) { - LOGGER.warn("Failed to apply operation", e); - } - - //Log warning message if event id increments are not sequential - if( event.getEventId() != (currentEventId + 1) ) { - LOGGER.warn("Received non-sequential event. currentEventId = {} received eventId = {} ", currentEventId, event.getEventId()); - } - currentEventId = event.getEventId(); - } - } - - LOGGER.info("Successfully fetched hive full snapshot, Current NotificationID: {}.", - currentEventId); - // As eventIDAfter is the last event that was processed, eventIDAfter is used to update - // lastProcessedNotificationID instead of getting it from persistent store. - return new PathsImage(pathsFullSnapshot, currentEventId, - SentryStore.EMPTY_PATHS_SNAPSHOT_ID); - } catch (TException failure) { - LOGGER.error("Fetching a new HMS snapshot cannot continue because an error occurred during " - + "the HMS communication: ", failure); - LOGGER.error("Root Exception", ExceptionUtils.getRootCause(failure)); - return new PathsImage(Collections.<String, Collection<String>>emptyMap(), - SentryStore.EMPTY_NOTIFICATION_ID, SentryStore.EMPTY_PATHS_SNAPSHOT_ID); - } - } - - /** - * Retrieve a Hive full snapshot from HMS. - * - * @return HMS snapshot. Snapshot consists of a mapping from auth object name to the set of paths - * corresponding to that name. - */ - private Map<String, Collection<String>> fetchFullUpdate() { - LOGGER.info("Request full HMS snapshot"); - try (FullUpdateInitializer updateInitializer = - new FullUpdateInitializer(hiveConnectionFactory, conf); - Context context = updateTimer.time()) { - Map<String, Collection<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot(); - LOGGER.info("Obtained full HMS snapshot"); - return pathsUpdate; - } catch (Exception ignored) { - failedSnapshotsCount.inc(); - LOGGER.error("Snapshot created failed ", ignored); - return emptyMap(); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java deleted file mode 100644 index efb8ae6..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.service.thrift; - -import java.io.File; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ThreadFactory; - -import javax.security.auth.Subject; -import javax.security.auth.kerberos.KerberosPrincipal; -import javax.security.auth.kerberos.KerberosTicket; -import javax.security.auth.login.LoginContext; -import javax.security.auth.login.LoginException; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Sets; - -public class SentryKerberosContext implements Runnable { - - private static final String KERBEROS_RENEWER_THREAD_NAME = "kerberos-renewer-%d"; - private static final float TICKET_RENEW_WINDOW = 0.80f; - private static final Logger LOGGER = LoggerFactory - .getLogger(SentryKerberosContext.class); - private LoginContext loginContext; - private Subject subject; - private final javax.security.auth.login.Configuration kerberosConfig; - private Thread renewerThread; - private boolean shutDownRenewer = false; - - public SentryKerberosContext(String principal, String keyTab, boolean server) - throws LoginException { - subject = new Subject(false, Sets.newHashSet(new KerberosPrincipal(principal)), - new HashSet<Object>(), new HashSet<Object>()); - if(server) { - kerberosConfig = KerberosConfiguration.createServerConfig(principal, new File(keyTab)); - } else { - kerberosConfig = KerberosConfiguration.createClientConfig(principal, new File(keyTab)); - } - loginWithNewContext(); - if (!server) { - startRenewerThread(); - } - } - - private void loginWithNewContext() throws LoginException { - LOGGER.info("Logging in with new Context"); - logoutSubject(); - loginContext = new LoginContext("", subject, null, kerberosConfig); - loginContext.login(); - subject = loginContext.getSubject(); - } - - private void logoutSubject() { - if (loginContext != null) { - try { - loginContext.logout(); - } catch (LoginException e) { - LOGGER.warn("Error logging out the subject", e); - } - } - loginContext = null; - } - - public Subject getSubject() { - return subject; - } - - /** - * Get the Kerberos TGT - * @return the user's TGT or null if none was found - */ - @Deprecated - private KerberosTicket getTGT() { - Set<KerberosTicket> tickets = subject.getPrivateCredentials(KerberosTicket.class); - for(KerberosTicket ticket: tickets) { - KerberosPrincipal server = ticket.getServer(); - if (server.getName().equals("krbtgt/" + server.getRealm() + - "@" + server.getRealm())) { - return ticket; - } - } - return null; - } - - private long getRefreshTime(KerberosTicket tgt) { - long start = tgt.getStartTime().getTime(); - long end = tgt.getEndTime().getTime(); - LOGGER.debug("Ticket start time: {}, end time: {}", start, end); - return start + (long) ((end - start) * TICKET_RENEW_WINDOW); - } - - /*** - * Ticket renewer thread - * wait till 80% time interval left on the ticket and then renew it - */ - @Override - public void run() { - try { - LOGGER.info("Sentry Ticket renewer thread started"); - while (!shutDownRenewer) { - KerberosTicket tgt = getTGT(); - if (tgt == null) { - LOGGER.warn("No ticket found in the cache"); - return; - } - long nextRefresh = getRefreshTime(tgt); - while (System.currentTimeMillis() < nextRefresh) { - Thread.sleep(1000); - if (shutDownRenewer) { - return; - } - } - loginWithNewContext(); - LOGGER.debug("Renewed ticket"); - } - } catch (InterruptedException e1) { - LOGGER.warn("Sentry Ticket renewer thread interrupted", e1); - return; - } catch (LoginException e) { - LOGGER.warn("Failed to renew ticket", e); - } finally { - logoutSubject(); - LOGGER.info("Sentry Ticket renewer thread finished"); - } - } - - public void startRenewerThread() { - ThreadFactory renewerThreadFactory = new ThreadFactoryBuilder() - .setNameFormat(KERBEROS_RENEWER_THREAD_NAME) - .build(); - renewerThread = renewerThreadFactory.newThread(this); - renewerThread.start(); - } - - public void shutDown() throws LoginException { - if (renewerThread != null) { - shutDownRenewer = true; - } else { - logoutSubject(); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java deleted file mode 100644 index d92ec21..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ /dev/null @@ -1,658 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.service.thrift; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.ServerSocket; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.EventListener; -import java.util.List; -import java.util.concurrent.*; - -import javax.security.auth.Subject; - -import com.codahale.metrics.Gauge; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SaslRpcServer; -import org.apache.hadoop.security.SaslRpcServer.AuthMethod; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.sentry.Command; -import org.apache.sentry.api.common.SentryServiceUtil; -import org.apache.sentry.core.common.utils.SigUtils; -import org.apache.sentry.provider.db.service.persistent.HMSFollower; -import org.apache.sentry.provider.db.service.persistent.LeaderStatusMonitor; -import org.apache.sentry.provider.db.service.persistent.SentryStore; -import org.apache.sentry.api.service.thrift.SentryHealthCheckServletContextListener; -import org.apache.sentry.api.service.thrift.SentryMetrics; -import org.apache.sentry.api.service.thrift.SentryMetricsServletContextListener; -import org.apache.sentry.api.service.thrift.SentryWebServer; -import org.apache.sentry.service.common.ServiceConstants; -import org.apache.sentry.service.common.ServiceConstants.ConfUtilties; -import org.apache.sentry.service.common.ServiceConstants.ServerConfig; -import org.apache.thrift.TMultiplexedProcessor; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.server.TServer; -import org.apache.thrift.server.TServerEventHandler; -import org.apache.thrift.server.TThreadPoolServer; -import org.apache.thrift.transport.TSaslServerTransport; -import org.apache.thrift.transport.TServerSocket; -import org.apache.thrift.transport.TServerTransport; -import org.apache.thrift.transport.TTransportFactory; -import org.eclipse.jetty.util.MultiException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Preconditions; - -import static org.apache.sentry.core.common.utils.SigUtils.registerSigListener; - -// Enable signal handler for HA leader/follower status if configured -public class SentryService implements Callable, SigUtils.SigListener { - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryService.class); - private HiveSimpleConnectionFactory hiveConnectionFactory; - - private static final String SENTRY_SERVICE_THREAD_NAME = "sentry-service"; - private static final String HMSFOLLOWER_THREAD_NAME = "hms-follower"; - private static final String STORE_CLEANER_THREAD_NAME = "store-cleaner"; - private static final String SERVICE_SHUTDOWN_THREAD_NAME = "service-shutdown"; - - private enum Status { - NOT_STARTED, - STARTED, - } - - private final Configuration conf; - private final InetSocketAddress address; - private final int maxThreads; - private final int minThreads; - private final boolean kerberos; - private final String principal; - private final String[] principalParts; - private final String keytab; - private final ExecutorService serviceExecutor; - private ScheduledExecutorService hmsFollowerExecutor = null; - private HMSFollower hmsFollower = null; - private Future serviceStatus; - private TServer thriftServer; - private Status status; - private final int webServerPort; - private SentryWebServer sentryWebServer; - private final long maxMessageSize; - /* - sentryStore provides the data access for sentry data. It is the singleton instance shared - between various {@link SentryPolicyService}, i.e., {@link SentryPolicyStoreProcessor} and - {@link HMSFollower}. - */ - private final SentryStore sentryStore; - private ScheduledExecutorService sentryStoreCleanService; - private final LeaderStatusMonitor leaderMonitor; - - public SentryService(Configuration conf) throws Exception { - this.conf = conf; - int port = conf - .getInt(ServerConfig.RPC_PORT, ServerConfig.RPC_PORT_DEFAULT); - if (port == 0) { - port = findFreePort(); - conf.setInt(ServerConfig.RPC_PORT, port); - } - this.address = NetUtils.createSocketAddr( - conf.get(ServerConfig.RPC_ADDRESS, ServerConfig.RPC_ADDRESS_DEFAULT), - port); - LOGGER.info("Configured on address {}", address); - kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase( - conf.get(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_KERBEROS).trim()); - maxThreads = conf.getInt(ServerConfig.RPC_MAX_THREADS, - ServerConfig.RPC_MAX_THREADS_DEFAULT); - minThreads = conf.getInt(ServerConfig.RPC_MIN_THREADS, - ServerConfig.RPC_MIN_THREADS_DEFAULT); - maxMessageSize = conf.getLong(ServerConfig.SENTRY_POLICY_SERVER_THRIFT_MAX_MESSAGE_SIZE, - ServerConfig.SENTRY_POLICY_SERVER_THRIFT_MAX_MESSAGE_SIZE_DEFAULT); - if (kerberos) { - // Use Hadoop libraries to translate the _HOST placeholder with actual hostname - try { - String rawPrincipal = Preconditions.checkNotNull(conf.get(ServerConfig.PRINCIPAL), ServerConfig.PRINCIPAL + " is required"); - principal = SecurityUtil.getServerPrincipal(rawPrincipal, address.getAddress()); - } catch(IOException io) { - throw new RuntimeException("Can't translate kerberos principal'", io); - } - LOGGER.info("Using kerberos principal: {}", principal); - - principalParts = SaslRpcServer.splitKerberosName(principal); - Preconditions.checkArgument(principalParts.length == 3, - "Kerberos principal should have 3 parts: " + principal); - keytab = Preconditions.checkNotNull(conf.get(ServerConfig.KEY_TAB), - ServerConfig.KEY_TAB + " is required"); - File keytabFile = new File(keytab); - Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(), - "Keytab %s does not exist or is not readable.", keytab); - } else { - principal = null; - principalParts = null; - keytab = null; - } - ThreadFactory sentryServiceThreadFactory = new ThreadFactoryBuilder() - .setNameFormat(SENTRY_SERVICE_THREAD_NAME) - .build(); - serviceExecutor = Executors.newSingleThreadExecutor(sentryServiceThreadFactory); - this.sentryStore = new SentryStore(conf); - sentryStore.setPersistUpdateDeltas(SentryServiceUtil.isHDFSSyncEnabled(conf)); - this.leaderMonitor = LeaderStatusMonitor.getLeaderStatusMonitor(conf); - webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT); - - status = Status.NOT_STARTED; - - // Enable signal handler for HA leader/follower status if configured - String sigName = conf.get(ServerConfig.SERVER_HA_STANDBY_SIG); - if ((sigName != null) && !sigName.isEmpty()) { - LOGGER.info("Registering signal handler {} for HA", sigName); - try { - registerSigListener(sigName, this); - } catch (Exception e) { - LOGGER.error("Failed to register signal", e); - } - } - } - - @Override - public String call() throws Exception { - SentryKerberosContext kerberosContext = null; - try { - status = Status.STARTED; - if (kerberos) { - kerberosContext = new SentryKerberosContext(principal, keytab, true); - Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<Void>() { - @Override - public Void run() throws Exception { - runServer(); - return null; - } - }); - } else { - runServer(); - } - } catch (Exception t) { - LOGGER.error("Error starting server", t); - throw new Exception("Error starting server", t); - } finally { - if (kerberosContext != null) { - kerberosContext.shutDown(); - } - status = Status.NOT_STARTED; - } - return null; - } - - private void runServer() throws Exception { - - startSentryStoreCleaner(conf); - startHMSFollower(conf); - - Iterable<String> processorFactories = ConfUtilties.CLASS_SPLITTER - .split(conf.get(ServerConfig.PROCESSOR_FACTORIES, - ServerConfig.PROCESSOR_FACTORIES_DEFAULT).trim()); - TMultiplexedProcessor processor = new TMultiplexedProcessor(); - boolean registeredProcessor = false; - for (String processorFactory : processorFactories) { - Class<?> clazz = conf.getClassByName(processorFactory); - if (!ProcessorFactory.class.isAssignableFrom(clazz)) { - throw new IllegalArgumentException("Processor Factory " - + processorFactory + " is not a " - + ProcessorFactory.class.getName()); - } - try { - Constructor<?> constructor = clazz - .getConstructor(Configuration.class); - LOGGER.info("ProcessorFactory being used: " + clazz.getCanonicalName()); - ProcessorFactory factory = (ProcessorFactory) constructor - .newInstance(conf); - boolean registerStatus = factory.register(processor, sentryStore); - if (!registerStatus) { - LOGGER.error("Failed to register " + clazz.getCanonicalName()); - } - registeredProcessor = registerStatus || registeredProcessor; - } catch (Exception e) { - throw new IllegalStateException("Could not create " - + processorFactory, e); - } - } - if (!registeredProcessor) { - throw new IllegalStateException( - "Failed to register any processors from " + processorFactories); - } - addSentryServiceGauge(); - TServerTransport serverTransport = new TServerSocket(address); - TTransportFactory transportFactory = null; - if (kerberos) { - TSaslServerTransport.Factory saslTransportFactory = new TSaslServerTransport.Factory(); - saslTransportFactory.addServerDefinition(AuthMethod.KERBEROS - .getMechanismName(), principalParts[0], principalParts[1], - ServerConfig.SASL_PROPERTIES, new GSSCallback(conf)); - transportFactory = saslTransportFactory; - } else { - transportFactory = new TTransportFactory(); - } - TThreadPoolServer.Args args = new TThreadPoolServer.Args( - serverTransport).processor(processor) - .transportFactory(transportFactory) - .protocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize)) - .minWorkerThreads(minThreads).maxWorkerThreads(maxThreads); - thriftServer = new TThreadPoolServer(args); - LOGGER.info("Serving on {}", address); - startSentryWebServer(); - - // thriftServer.serve() does not return until thriftServer is stopped. Need to log before - // calling thriftServer.serve() - LOGGER.info("Sentry service is ready to serve client requests"); - - // Allow clients/users watching the console to know when sentry is ready - System.out.println("Sentry service is ready to serve client requests"); - SentryStateBank.enableState(SentryServiceState.COMPONENT, SentryServiceState.SERVICE_RUNNING); - thriftServer.serve(); - } - - private void startHMSFollower(Configuration conf) throws Exception { - boolean syncPolicyStore = SentryServiceUtil.isSyncPolicyStoreEnabled(conf); - - if ((!SentryServiceUtil.isHDFSSyncEnabled(conf)) && (!syncPolicyStore)) { - LOGGER.info("HMS follower is not started because HDFS sync is disabled and perm sync is disabled"); - return; - } - - String metastoreURI = SentryServiceUtil.getHiveMetastoreURI(); - if (metastoreURI == null) { - LOGGER.info("Metastore uri is not configured. Do not start HMSFollower"); - return; - } - - LOGGER.info("Starting HMSFollower to HMS {}", metastoreURI); - - Preconditions.checkState(hmsFollower == null); - Preconditions.checkState(hmsFollowerExecutor == null); - Preconditions.checkState(hiveConnectionFactory == null); - - hiveConnectionFactory = new HiveSimpleConnectionFactory(conf, new HiveConf()); - hiveConnectionFactory.init(); - hmsFollower = new HMSFollower(conf, sentryStore, leaderMonitor, hiveConnectionFactory); - long initDelay = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS, - ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT); - long period = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, - ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT); - try { - ThreadFactory hmsFollowerThreadFactory = new ThreadFactoryBuilder() - .setNameFormat(HMSFOLLOWER_THREAD_NAME) - .build(); - hmsFollowerExecutor = Executors.newScheduledThreadPool(1, hmsFollowerThreadFactory); - hmsFollowerExecutor.scheduleAtFixedRate(hmsFollower, - initDelay, period, TimeUnit.MILLISECONDS); - } catch (IllegalArgumentException e) { - LOGGER.error(String.format("Could not start HMSFollower due to illegal argument. period is %s ms", - period), e); - throw e; - } - } - - private void stopHMSFollower(Configuration conf) { - if ((hmsFollowerExecutor == null) || (hmsFollower == null)) { - Preconditions.checkState(hmsFollower == null); - Preconditions.checkState(hmsFollowerExecutor == null); - - LOGGER.debug("Skip shuting down hmsFollowerExecutor and closing hmsFollower because they are not created"); - return; - } - - Preconditions.checkNotNull(hmsFollowerExecutor); - Preconditions.checkNotNull(hmsFollower); - Preconditions.checkNotNull(hiveConnectionFactory); - - // use follower scheduling interval as timeout for shutting down its executor as - // such scheduling interval should be an upper bound of how long the task normally takes to finish - long timeoutValue = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, - ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT); - try { - SentryServiceUtil.shutdownAndAwaitTermination(hmsFollowerExecutor, "hmsFollowerExecutor", - timeoutValue, TimeUnit.MILLISECONDS, LOGGER); - } finally { - try { - hiveConnectionFactory.close(); - } catch (Exception e) { - LOGGER.error("Can't close HiveConnectionFactory", e); - } - hmsFollowerExecutor = null; - hiveConnectionFactory = null; - try { - // close connections - hmsFollower.close(); - } catch (Exception ex) { - LOGGER.error("HMSFollower.close() failed", ex); - } finally { - hmsFollower = null; - } - } - } - - private void startSentryStoreCleaner(Configuration conf) { - Preconditions.checkState(sentryStoreCleanService == null); - - // If SENTRY_STORE_CLEAN_PERIOD_SECONDS is set to positive, the background SentryStore cleaning - // thread is enabled. Currently, it only purges the delta changes {@link MSentryChange} in - // the sentry store. - long storeCleanPeriodSecs = conf.getLong( - ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS, - ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS_DEFAULT); - if (storeCleanPeriodSecs <= 0) { - return; - } - - try { - Runnable storeCleaner = new Runnable() { - @Override - public void run() { - if (leaderMonitor.isLeader()) { - sentryStore.purgeDeltaChangeTables(); - sentryStore.purgeNotificationIdTable(); - } - } - }; - - ThreadFactory sentryStoreCleanerThreadFactory = new ThreadFactoryBuilder() - .setNameFormat(STORE_CLEANER_THREAD_NAME) - .build(); - sentryStoreCleanService = Executors.newSingleThreadScheduledExecutor(sentryStoreCleanerThreadFactory); - sentryStoreCleanService.scheduleWithFixedDelay( - storeCleaner, 0, storeCleanPeriodSecs, TimeUnit.SECONDS); - - LOGGER.info("sentry store cleaner is scheduled with interval {} seconds", storeCleanPeriodSecs); - } - catch(IllegalArgumentException e){ - LOGGER.error("Could not start SentryStoreCleaner due to illegal argument", e); - sentryStoreCleanService = null; - } - } - - private void stopSentryStoreCleaner() { - Preconditions.checkNotNull(sentryStoreCleanService); - - try { - SentryServiceUtil.shutdownAndAwaitTermination(sentryStoreCleanService, "sentryStoreCleanService", - 10, TimeUnit.SECONDS, LOGGER); - } - finally { - sentryStoreCleanService = null; - } - } - - private void addSentryServiceGauge() { - SentryMetrics.getInstance().addSentryServiceGauges(this); - } - - private void startSentryWebServer() throws Exception{ - Boolean sentryReportingEnable = conf.getBoolean(ServerConfig.SENTRY_WEB_ENABLE, - ServerConfig.SENTRY_WEB_ENABLE_DEFAULT); - if(sentryReportingEnable) { - List<EventListener> listenerList = new ArrayList<>(); - listenerList.add(new SentryHealthCheckServletContextListener()); - listenerList.add(new SentryMetricsServletContextListener()); - sentryWebServer = new SentryWebServer(listenerList, webServerPort, conf); - sentryWebServer.start(); - } - } - - private void stopSentryWebServer() throws Exception{ - if( sentryWebServer != null) { - sentryWebServer.stop(); - sentryWebServer = null; - } - } - - public InetSocketAddress getAddress() { - return address; - } - - public synchronized boolean isRunning() { - return status == Status.STARTED && thriftServer != null - && thriftServer.isServing(); - } - - public synchronized void start() throws Exception{ - if (status != Status.NOT_STARTED) { - throw new IllegalStateException("Cannot start when " + status); - } - LOGGER.info("Attempting to start..."); - serviceStatus = serviceExecutor.submit(this); - } - - public synchronized void stop() throws Exception{ - MultiException exception = null; - LOGGER.info("Attempting to stop..."); - leaderMonitor.close(); - if (isRunning()) { - LOGGER.info("Attempting to stop sentry thrift service..."); - try { - thriftServer.stop(); - thriftServer = null; - status = Status.NOT_STARTED; - } catch (Exception e) { - LOGGER.error("Error while stopping sentry thrift service", e); - exception = addMultiException(exception,e); - } - } else { - thriftServer = null; - status = Status.NOT_STARTED; - LOGGER.info("Sentry thrift service is already stopped..."); - } - if (isWebServerRunning()) { - try { - LOGGER.info("Attempting to stop sentry web service..."); - stopSentryWebServer(); - } catch (Exception e) { - LOGGER.error("Error while stopping sentry web service", e); - exception = addMultiException(exception,e); - } - } else { - LOGGER.info("Sentry web service is already stopped..."); - } - - stopHMSFollower(conf); - stopSentryStoreCleaner(); - - if (exception != null) { - exception.ifExceptionThrow(); - } - SentryStateBank.disableState(SentryServiceState.COMPONENT,SentryServiceState.SERVICE_RUNNING); - LOGGER.info("Stopped..."); - } - - /** - * If the current daemon is active, make it standby. - * Here 'active' means it is the only daemon that can fetch snapshots from HMA and write - * to the backend DB. - */ - @VisibleForTesting - public synchronized void becomeStandby() { - leaderMonitor.deactivate(); - } - - private MultiException addMultiException(MultiException exception, Exception e) { - MultiException newException = exception; - if (newException == null) { - newException = new MultiException(); - } - newException.add(e); - return newException; - } - - private boolean isWebServerRunning() { - return sentryWebServer != null - && sentryWebServer.isAlive(); - } - - private static int findFreePort() { - int attempts = 0; - while (attempts++ <= 1000) { - try { - ServerSocket s = new ServerSocket(0); - int port = s.getLocalPort(); - s.close(); - return port; - } catch (IOException e) { - // ignore and retry - } - } - throw new IllegalStateException("Unable to find a port after 1000 attempts"); - } - - public static Configuration loadConfig(String configFileName) - throws MalformedURLException { - File configFile = null; - if (configFileName == null) { - throw new IllegalArgumentException("Usage: " - + ServiceConstants.ServiceArgs.CONFIG_FILE_LONG - + " path/to/sentry-service.xml"); - } else if (!((configFile = new File(configFileName)).isFile() && configFile - .canRead())) { - throw new IllegalArgumentException("Cannot read configuration file " - + configFile); - } - Configuration conf = new Configuration(false); - conf.addResource(configFile.toURI().toURL(), true); - return conf; - } - - public static class CommandImpl implements Command { - @Override - public void run(String[] args) throws Exception { - CommandLineParser parser = new GnuParser(); - Options options = new Options(); - options.addOption(ServiceConstants.ServiceArgs.CONFIG_FILE_SHORT, - ServiceConstants.ServiceArgs.CONFIG_FILE_LONG, - true, "Sentry Service configuration file"); - CommandLine commandLine = parser.parse(options, args); - String configFileName = commandLine.getOptionValue(ServiceConstants. - ServiceArgs.CONFIG_FILE_LONG); - File configFile = null; - if (configFileName == null || commandLine.hasOption("h") || commandLine.hasOption("help")) { - // print usage - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("sentry --command service", options); - System.exit(-1); - } else if(!((configFile = new File(configFileName)).isFile() && configFile.canRead())) { - throw new IllegalArgumentException("Cannot read configuration file " + configFile); - } - Configuration serverConf = loadConfig(configFileName); - final SentryService server = new SentryService(serverConf); - server.start(); - - ThreadFactory serviceShutdownThreadFactory = new ThreadFactoryBuilder() - .setNameFormat(SERVICE_SHUTDOWN_THREAD_NAME) - .build(); - Runtime.getRuntime().addShutdownHook(serviceShutdownThreadFactory.newThread(new Runnable() { - @Override - public void run() { - LOGGER.info("ShutdownHook shutting down server"); - try { - server.stop(); - } catch (Throwable t) { - LOGGER.error("Error stopping SentryService", t); - System.exit(1); - } - } - })); - - // Let's wait on the service to stop - try { - // Wait for the service thread to finish - server.serviceStatus.get(); - } finally { - server.serviceExecutor.shutdown(); - } - } - } - - public Configuration getConf() { - return conf; - } - - /** - * Add Thrift event handler to underlying thrift threadpool server - * @param eventHandler - */ - public void setThriftEventHandler(TServerEventHandler eventHandler) throws IllegalStateException { - if (thriftServer == null) { - throw new IllegalStateException("Server is not initialized or stopped"); - } - thriftServer.setServerEventHandler(eventHandler); - } - - public TServerEventHandler getThriftEventHandler() throws IllegalStateException { - if (thriftServer == null) { - throw new IllegalStateException("Server is not initialized or stopped"); - } - return thriftServer.getEventHandler(); - } - - public Gauge<Boolean> getIsActiveGauge() { - return new Gauge<Boolean>() { - @Override - public Boolean getValue() { - return leaderMonitor.isLeader(); - } - }; - } - - public Gauge<Long> getBecomeActiveCount() { - return new Gauge<Long>() { - @Override - public Long getValue() { - return leaderMonitor.getLeaderCount(); - } - }; - } - - @Override - public void onSignal(String signalName) { - // Become follower - leaderMonitor.deactivate(); - } - - /** - * Restart HMSFollower with new configuration - * @param newConf Configuration - * @throws Exception - */ - @VisibleForTesting - public void restartHMSFollower(Configuration newConf) throws Exception{ - stopHMSFollower(conf); - startHMSFollower(newConf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java deleted file mode 100644 index d146a0d..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.service.thrift; - -import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.core.common.transport.RetryClientInvocationHandler; -import org.apache.sentry.core.common.transport.SentryPolicyClientTransportConfig; -import org.apache.sentry.core.common.transport.SentryTransportFactory; -import org.apache.sentry.core.common.transport.SentryTransportPool; -import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient; -import org.apache.sentry.api.service.thrift.SentryPolicyServiceClientDefaultImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.concurrent.ThreadSafe; -import java.lang.reflect.Proxy; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Client factory for Hive clients. The factory uses connection pooling. - */ -@ThreadSafe -public final class SentryServiceClientFactory { - private static final Logger LOGGER = LoggerFactory.getLogger(SentryServiceClientFactory.class); - - private static final SentryPolicyClientTransportConfig transportConfig = - new SentryPolicyClientTransportConfig(); - private final Configuration conf; - private final SentryTransportPool transportPool; - - /** Keep track of singleton instances */ - private static final AtomicReference<SentryServiceClientFactory> clientFactory = - new AtomicReference<>(); - - /** - * Create a client instance. The supplied configuration is only used the first time and - * ignored afterwords. Tests that want to supply different configurations - * should call {@link #factoryReset(SentryServiceClientFactory)} to force new configuration - * read. - * @param conf Configuration - * @return client instance - * @throws Exception - */ - public static SentryPolicyServiceClient create(Configuration conf) throws Exception { - SentryServiceClientFactory factory = clientFactory.get(); - if (factory != null) { - return factory.create(); - } - factory = new SentryServiceClientFactory(conf); - boolean ok = clientFactory.compareAndSet(null, factory); - if (ok) { - return factory.create(); - } - // Close old factory - factory.close(); - return clientFactory.get().create(); - } - - /** - * Create a new instance of the factory which will hand hand off connections from - * the pool. - * @param conf Configuration object - */ - private SentryServiceClientFactory(Configuration conf) { - this.conf = conf; - - transportPool = new SentryTransportPool(conf, transportConfig, - new SentryTransportFactory(conf, transportConfig)); - } - - private SentryPolicyServiceClient create() throws Exception { - return (SentryPolicyServiceClient) Proxy - .newProxyInstance(SentryPolicyServiceClientDefaultImpl.class.getClassLoader(), - SentryPolicyServiceClientDefaultImpl.class.getInterfaces(), - new RetryClientInvocationHandler(conf, - new SentryPolicyServiceClientDefaultImpl(conf, transportPool), transportConfig)); - } - - /** - * Reset existing factory and return the old one. - * Only used by tests. - */ - public static SentryServiceClientFactory factoryReset(SentryServiceClientFactory factory) { - LOGGER.debug("factory reset"); - return clientFactory.getAndSet(factory); - } - - public void close() { - try { - transportPool.close(); - } catch (Exception e) { - LOGGER.error("failed to close transport pool", e); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java deleted file mode 100644 index c1d81ed..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.service.thrift; - -import org.apache.hadoop.conf.Configuration; - -public class SentryServiceFactory { - public static SentryService create(Configuration conf) throws Exception { - return new SentryService(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java deleted file mode 100644 index 4219adc..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.sentry.service.thrift; - -/** - * States for the SentryService - */ -public enum SentryServiceState implements SentryState { - /** - * The SentryService is running all of its threads and services. This include the store cleaner, - * the web interface, the HMS poller, and the Thrift Server - */ - SERVICE_RUNNING, - - /** - * A full update of data from the HMS is running by the thread handling the update. - */ - FULL_UPDATE_RUNNING; - - /** - * The component name this state is for. - */ - public static final String COMPONENT = "SentryService"; - - /** - * {@inheritDoc} - */ - @Override - public long getValue() { - return 1 << this.ordinal(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java deleted file mode 100644 index 040d82a..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.sentry.service.thrift; - -/** - * Interface for SentryState enums. - */ -public interface SentryState { - - /** - * This gets the Bitmask value associated with the state. - */ - long getValue(); -} http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java deleted file mode 100644 index 2c05d49..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.sentry.service.thrift; - - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.AtomicLongMap; -import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.concurrent.ThreadSafe; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * <p>SentryStateBank is a state visibility manager to allow components to communicate state to other - * parts of the application.</p> - * - * <p>It allows you to provide multiple boolean states for a component and expose those states to - * other parts of the application without having references to the actual instances of the classes - * setting those states.</p> - * - * <p>SentryStateBank uses a bitmasked long in order to store the states, so its very compact and - * efficient.</p> - * - * <p>States are defined using an enum that implements the {@link SentryState} interface. The - * {@link SentryState} implementation can provide up to 64 states per components. The {@link SentryState#getValue()} - * implementation should return a bitshift of the oridinal of the enum value. This gives the bitmask - * location to be checking for the state.</p> - * - * <p>The following is an example of a simple {@link SentryState} enum implementation</p> - * - * <pre> - * {@code - * - * public enum ExampleState implements SentryState { - * FIRST_STATE, - * SECOND_STATE; - * - * public static final String COMPONENT = "ExampleState"; - * - * @Override - * public long getValue() { - * return 1 << this.ordinal(); - * } - * } - * } - * </pre> - * - * <p>This class is thread safe. It uses a {@link ReentrantReadWriteLock} to wrap accesses and changes - * to the state.</p> - */ -@ThreadSafe -public final class SentryStateBank { - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryStateBank.class); - private static final AtomicLongMap<String> states = AtomicLongMap.create(); - private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - protected SentryStateBank() { - } - - @VisibleForTesting - static void clearAllStates() { - states.clear(); - LOGGER.debug("All states have been cleared."); - } - - @VisibleForTesting - static void resetComponentState(String component) { - states.remove(component); - LOGGER.debug("All states have been cleared for component {}", component); - } - - /** - * Enables a state - * - * @param component the component for the state - * @param state the state to disable - */ - public static void enableState(String component, SentryState state) { - lock.writeLock().lock(); - try { - states.put(component, states.get(component) | state.getValue()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("{} entered state {}", component, state.toString()); - } - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Disables a state for a component - * - * @param component the component for the state - * @param state the state to disable - */ - public static void disableState(String component, SentryState state) { - lock.writeLock().lock(); - try { - states.put(component, states.get(component) & (~state.getValue())); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("{} exited state {}", component, state.toString()); - } - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Returns if a state is enabled or not - * - * @param component The component for the state - * @param state the SentryState to check - * @return true if the state for the component is enabled - */ - public static boolean isEnabled(String component, SentryState state) { - lock.readLock().lock(); - try { - return (states.get(component) & state.getValue()) == state.getValue(); - } finally { - lock.readLock().unlock(); - } - - } - - /** - * Checks if all of the states passed in are enabled - * - * @param component The component for the states - * @param passedStates the SentryStates to check - */ - public static boolean hasStatesEnabled(String component, Set<SentryState> passedStates) { - lock.readLock().lock(); - try { - long value = 0L; - - for (SentryState state : passedStates) { - value += state.getValue(); - } - return (states.get(component) & value) == value; - } finally { - lock.readLock().unlock(); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html b/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html deleted file mode 100644 index 9f52a8e..0000000 --- a/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html +++ /dev/null @@ -1,61 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> -<!DOCTYPE HTML> -<html lang="en"> - <head> - <meta charset="utf-8"> - <title>Sentry Service</title> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <meta name="description" content=""> - <link href="css/bootstrap.min.css" rel="stylesheet"> - <link href="css/bootstrap-theme.min.css" rel="stylesheet"> - <link href="css/sentry.css" rel="stylesheet"> - </head> - - <body> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <div class="navbar-header"> - <a class="navbar-brand" href="#"><img src="sentry.png" alt="Sentry Logo"/></a> - </div> - <div class="collapse navbar-collapse"> - <ul class="nav navbar-nav"> - <li class="active"><a href="#">Home</a></li> - <li><a href="/metrics?pretty=true">Metrics</a></li> - <li><a href="/threads">Threads</a></li> - <li><a href="/conf">Configuration</a></li> - </ul> - </div> - </div> - </nav> - - <div class="container"> - <div class="page-header"><h2>Sentry Service</h2></div> - <ul> - <li><a href="/metrics?pretty=true">Metrics</a></li> - <li><a href="/threads">Threads</a></li> - <li><a href="/conf">Configuration</a></li> - </ul> - </div> - - <footer class="footer"> - <div class="container"> - <p class="text-muted">SENTRY 2.0.0-SNAPSHOT</p> - </div> - </footer> - </body> -</html>
