Repository: sentry Updated Branches: refs/heads/master 441a4f64a -> 73a5eabea
SENTRY-1520: Provide mechanism for triggering HMS full snapshot (Vadim Spector, Reviewed by: Sravya Tirukkovalur, Alexander Kolbasov, Kalyan Kalvagadda and Hao Hao) Change-Id: I026e55a464c8967a6d41a4aff40ad902c52fc090 Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/73a5eabe Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/73a5eabe Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/73a5eabe Branch: refs/heads/master Commit: 73a5eabeaef5f9b76a9366bc74d5cbd99a32bc3f Parents: 441a4f6 Author: hahao <hao....@cloudera.com> Authored: Tue Nov 22 15:51:23 2016 -0800 Committer: hahao <hao....@cloudera.com> Committed: Tue Nov 22 15:54:38 2016 -0800 ---------------------------------------------------------------------- .../sentry/core/common/utils/SigUtils.java | 97 +++++++++++ .../apache/sentry/hdfs/ServiceConstants.java | 1 + .../org/apache/sentry/hdfs/SentryPlugin.java | 167 ++++++++++++++++++- 3 files changed, 260 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/73a5eabe/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/utils/SigUtils.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/utils/SigUtils.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/utils/SigUtils.java new file mode 100644 index 0000000..1f16e14 --- /dev/null +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/utils/SigUtils.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.core.common.utils; + +import sun.misc.Signal; +import sun.misc.SignalHandler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class facilitates handling signals in Sentry. + * <p> + * It defines its own callback interface to isolate the client code + * from the dependency on sun.misc.*. The sun.misc.* package is not officially + * supported, yet, per many indications, it is very unlikely to be dropped in + * the future JDK releases. If needed it can be re-implemented using JNI, + * though we'd rather not ever take this path. + * <p> + * This class relies on sun.misc.SignalHandler which registers signal handlers via static + * method handle(). Therefore, SigUtils also only exposes a static method registerListener() + * <p> + * This class, as sun.misc.SignalHandler, supports a single signal listener per each signal. + * The same signal listener can be used to handle multiple signals, but setting a listener + * for the same signal twice will replace an old listener with the new one. + * method. + */ +public final class SigUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(SigUtils.class); + + private SigUtils() {} // to prevent class instantiation + + /** + * interface to be implemented by a class that wants to be notified of a signal + */ + public interface SigListener { + void onSignal(String signalName); + } + + /* private wrapper class around SignalHandler, to delegate signal handling + * to a registered signal listener. + */ + private static class SigHandler implements SignalHandler { + private final SigListener listener; + SigHandler(SigListener listener) { + this.listener = listener; + } + @Override + public void handle(Signal sig) { + if (sig != null) { + listener.onSignal(sig.toString()); + // signalling is a rare yet important event - let's always log it + LOGGER.info("Signal propagated: " + sig); + } else { + LOGGER.error("Internal Error: null signal received"); + } + } + } + + /** + * Register signal listener for a specific signal. + * + * Only one listener per specific signal is supported. Calling this method + * multiple times will keep overwriting a previously set listener. + * + * @NotNull sigName + * @NotNull sigListener + * @param sigName signal name, as in UNIX "kill" command, e.g. INT, USR1, etc, without "SIG" prefix + * @param sigListener signal notification callback object + * @throws IllegalArgumentException if invalid signal name or a signal is already handled by OS or JVM + */ + public static void registerSigListener(String sigName, SigListener sigListener) { + if (sigListener == null) { + throw new IllegalArgumentException("NULL signal listener"); + } + if (sigName == null) { + throw new IllegalArgumentException("NULL signal name"); + } + Signal.handle(new Signal(sigName), new SigHandler(sigListener)); + LOGGER.info("Signal Listener registered for signal " + sigName); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/73a5eabe/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java index 2c0ae82..23552c2 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java @@ -66,6 +66,7 @@ public class ServiceConstants { public static final int SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC_DEFAULT = 100; public static final String SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC = "sentry.hdfs.sync.metastore.cache.max-tables-per-rpc"; public static final int SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC_DEFAULT = 100; + public static final String SENTRY_SERVICE_FULL_UPDATE_SIGNAL = "sentry.hdfs.sync.full-update-signal"; } public static class ClientConfig { http://git-wip-us.apache.org/repos/asf/sentry/blob/73a5eabe/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index f3926a2..3695709 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -22,10 +22,12 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import com.codahale.metrics.Timer; import org.apache.hadoop.conf.Configuration; +import org.apache.sentry.core.common.utils.SigUtils; import org.apache.sentry.hdfs.ServiceConstants.ServerConfig; import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; @@ -46,10 +48,74 @@ import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SentryPlugin implements SentryPolicyStorePlugin { + /** + * SentryPlugin facilitates HDFS synchronization between HMS and NameNode. + * <p> + * Normally, synchronization happens via partial (incremental) updates: + * <ol> + * <li> + * Whenever updates happen on HMS, they are immediately pushed to Sentry. + * Commonly, it's a single update per remote call. + * <li> + * The NameNode periodically asks Sentry for updates. Sentry may return zero + * or more updates previously received from HMS. + * </ol> + * <p> + * Each individual update is assigned a corresponding sequence number. Those + * numbers serve to detect the out-of-sync situations between HMS and Sentry and + * between Sentry and NameNode. Detecting out-of-sync situation triggers full + * update between the components that are out-of-sync. + * <p> + * SentryPlugin also implements signal-triggered mechanism of full path + * updates from HMS to Sentry and from Sentry to NameNode, to address + * mission-critical out-of-sync situations that may be encountered in the field. + * Those out-of-sync situations may not be detectable via the exsiting sequence + * numbers mechanism (most likely due to the implementation bugs). + * <p> + * To facilitate signal-triggered full update from HMS to Sentry and from Sentry + * to the NameNode, the following 3 boolean variables are defined: + * fullUpdateHMS, fullUpdateHMSWait, and fullUpdateNN. + * <ol> + * <li> + * The purpose of fullUpdateHMS is to ensure that Sentry asks HMS for full + * update, and does so only once per signal. + * <li> + * The purpose of fullUpdateNN is to ensure that Sentry sends full update + * to NameNode, and does so only once per signal. + * <li> + * The purpose of fullUpdateHMSWait is to ensure that NN update only happens + * after HMS update. + * </ol> + * The details: + * <ol> + * <li> + * Upon receiving a signal, fullUpdateHMS, fullUpdateHMSWait, and fullUpdateNN + * are all set to true. + * <li> + * On the next call to getLastSeenHMSPathSeqNum() from HMS, Sentry checks if + * fullUpdateHMS == true. If yes, it returns invalid (zero) sequence number + * to HMS, so HMS would push full update by calling handlePathUpdateNotification() + * next time. fullUpdateHMS is immediately reset to false, to only trigger one + * full update request to HMS per signal. + * <li> + * When HMS calls handlePathUpdateNotification(), Sentry checks if the update + * is a full image. If it is, fullUpdateHMSWait is set to false. + * <li> + * When NameNode calls getAllPathsUpdatesFrom() asking for partial update, + * Sentry checks if both fullUpdateNN == true and fullUpdateHMSWait == false. + * If yes, it sends full update back to NameNode and immediately resets + * fullUpdateNN to false. + * </ol> + */ + +public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListener { private static final Logger LOGGER = LoggerFactory.getLogger(SentryPlugin.class); + private final AtomicBoolean fullUpdateHMSWait = new AtomicBoolean(false); + private final AtomicBoolean fullUpdateHMS = new AtomicBoolean(false); + private final AtomicBoolean fullUpdateNN = new AtomicBoolean(false); + public static volatile SentryPlugin instance; static class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> { @@ -98,9 +164,25 @@ public class SentryPlugin implements SentryPolicyStorePlugin { private final AtomicLong permSeqNum = new AtomicLong(5); private PermImageRetriever permImageRetriever; private boolean outOfSync = false; - + /* + * This number is smaller than starting sequence numbers used by NN and HMS + * so in both cases its effect is to creat appearence of out-of-sync + * updates on the Sentry server (as if there were no previous updates at all). + * It, in turn, triggers a) pushing full update from HMS to Sentry and + * b) pulling full update from Sentry to NameNode. + */ + private static final long NO_LAST_SEEN_HMS_PATH_SEQ_NUM = 0L; + + /* + * Call from HMS to get the last known update sequence #. + */ long getLastSeenHMSPathSeqNum() { - return pathsUpdater.getLastSeen(); + if (!fullUpdateHMS.getAndSet(false)) { + return pathsUpdater.getLastSeen(); + } else { + LOGGER.info("SIGNAL HANDLING: asking for full update from HMS"); + return NO_LAST_SEEN_HMS_PATH_SEQ_NUM; + } } @Override @@ -120,20 +202,87 @@ public class SentryPlugin implements SentryPolicyStorePlugin { permImageRetriever, 100, initUpdateRetryDelayMs); LOGGER.info("Sentry HDFS plugin initialized !!"); instance = this; + + // register signal handler(s) if any signal(s) are configured + String[] sigs = conf.getStrings(ServerConfig.SENTRY_SERVICE_FULL_UPDATE_SIGNAL, null); + if (sigs != null && sigs.length != 0) { + for (String sig : sigs) { + try { + LOGGER.info("SIGNAL HANDLING: Registering Signal Handler For " + sig); + SigUtils.registerSigListener(sig, this); + } catch (Exception e) { + LOGGER.error("SIGNAL HANDLING: Signal Handle Registration Failure", e); + } + } + } } + /** + * Request for update from NameNode. + * Full update to NameNode should happen only after full update from HMS. + */ public List<PathsUpdate> getAllPathsUpdatesFrom(long pathSeqNum) { - return pathsUpdater.getAllUpdatesFrom(pathSeqNum); + if (!fullUpdateNN.get()) { + // Most common case - Sentry is NOT handling a full update. + return pathsUpdater.getAllUpdatesFrom(pathSeqNum); + } else if (!fullUpdateHMSWait.get()) { + /* + * Sentry is in the middle of signal-triggered full update. + * It already got a full update from HMS + */ + LOGGER.info("SIGNAL HANDLING: sending full update to NameNode"); + fullUpdateNN.set(false); // don't do full NN update till the next signal + List<PathsUpdate> updates = pathsUpdater.getAllUpdatesFrom(NO_LAST_SEEN_HMS_PATH_SEQ_NUM); + /* + * This code branch is only called when Sentry is in the middle of a full update + * (fullUpdateNN == true) and Sentry has already received full update from HMS + * (fullUpdateHMSWait == false). It means that the local cache has a full update + * from HMS. + * + * The full update list is expected to contain the last full update as the first + * element, followed by zero or more subsequent partial updates. + * + * Returning NULL, empty, or partial update instead would be unexplainable, so + * it should be logged. + */ + if (updates != null) { + if (!updates.isEmpty()) { + if (updates.get(0).hasFullImage()) { + LOGGER.info("SIGNAL HANDLING: Confirmed full update to NameNode"); + } else { + LOGGER.warn("SIGNAL HANDLING: Sending partial instead of full update to NameNode (???)"); + } + } else { + LOGGER.warn("SIGNAL HANDLING: Sending empty instead of full update to NameNode (???)"); + } + } else { + LOGGER.warn("SIGNAL HANDLING: returned NULL instead of full update to NameNode (???)"); + } + return updates; + } else { + // Sentry is handling a full update, but not yet received full update from HMS + LOGGER.warn("SIGNAL HANDLING: sending partial update to NameNode: still waiting for full update from HMS"); + return pathsUpdater.getAllUpdatesFrom(pathSeqNum); + } } public List<PermissionsUpdate> getAllPermsUpdatesFrom(long permSeqNum) { return permsUpdater.getAllUpdatesFrom(permSeqNum); } + /* + * Handle partial (most common) or full update from HMS + */ public void handlePathUpdateNotification(PathsUpdate update) throws SentryPluginException { pathsUpdater.handleUpdateNotification(update); - LOGGER.debug("Recieved Authz Path update [" + update.getSeqNum() + "].."); + if (!update.hasFullImage()) { // most common case of partial update + LOGGER.debug("Recieved Authz Path update [" + update.getSeqNum() + "].."); + } else { // rare case of full update + LOGGER.warn("Recieved Authz Path FULL update [" + update.getSeqNum() + "].."); + // indicate that we're ready to send full update to NameNode + fullUpdateHMSWait.set(false); + } } @Override @@ -256,6 +405,14 @@ public class SentryPlugin implements SentryPolicyStorePlugin { LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); } + @Override + public void onSignal(final String sigName) { + LOGGER.info("SIGNAL HANDLING: Received signal " + sigName + ", triggering full update"); + fullUpdateHMS.set(true); + fullUpdateHMSWait.set(true); + fullUpdateNN.set(true); + } + private String getAuthzObj(TSentryPrivilege privilege) { String authzObj = null; if (!SentryStore.isNULL(privilege.getDbName())) {
