Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 731a5d15e -> 8f5c17dbc
SENTRY-1566: Make full Perm/Path snapshot available for NN plugin (Hao Hao, Reviewed by: Alexander Kolbasov and Lei Xu) Change-Id: I2788046192b71a3988f5d8bf1fa903eff18a7c8d Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/8f5c17db Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/8f5c17db Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/8f5c17db Branch: refs/heads/sentry-ha-redesign Commit: 8f5c17dbce6417b0c98ba6c2c31fa9c7fd8b222f Parents: 731a5d1 Author: hahao <[email protected]> Authored: Thu Mar 9 15:37:38 2017 -0800 Committer: hahao <[email protected]> Committed: Fri Mar 10 11:14:17 2017 -0800 ---------------------------------------------------------------------- .../org/apache/sentry/hdfs/ImageRetriever.java | 45 ++++ .../org/apache/sentry/hdfs/PathsUpdate.java | 3 +- .../apache/sentry/hdfs/PathImageRetriever.java | 72 ++++++ .../apache/sentry/hdfs/PermImageRetriever.java | 93 ++++++++ .../sentry/hdfs/SentryHdfsMetricsUtil.java | 27 ++- .../org/apache/sentry/hdfs/SentryPlugin.java | 53 +---- .../org/apache/sentry/hdfs/UpdateForwarder.java | 32 ++- .../sentry/hdfs/UpdateablePermissions.java | 7 +- .../apache/sentry/hdfs/TestUpdateForwarder.java | 15 +- .../db/service/persistent/PathsImage.java | 48 ++++ .../db/service/persistent/PermissionsImage.java | 57 +++++ .../db/service/persistent/SentryStore.java | 219 ++++++++++++------- .../db/service/persistent/TestSentryStore.java | 56 ++++- 13 files changed, 545 insertions(+), 182 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java new file mode 100644 index 0000000..0e40756 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import static org.apache.sentry.hdfs.Updateable.Update; + +/** + * ImageRetriever obtains a complete snapshot of either Sentry Permissions + * ({@code PermissionsUpdate}) or Sentry representation of Hive Paths + * ({@code PathsUpdate}). + * <p> + * The snapshot image should represent a consistent state. + * The {@link #retrieveFullImage(long)} method obtains such state snapshot from + * a persistent storage. + * The Snapshots are propagated to a consumer of Sentry, such as HDFS NameNode, + * whenever the consumer needs to synchronize its full state. + */ +public interface ImageRetriever<K extends Update> { + + /** + * Retrieve a complete snapshot of type {@code Update} from a persistent storage. + * + * @param seqNum + * @return a complete snapshot of type {@link Update}, e.g {@link PermissionsUpdate} + * or {@link PathsUpdate} + * @throws Exception + */ + K retrieveFullImage(long seqNum) throws Exception; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index 992c8b7..14e967a 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -39,7 +39,6 @@ import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * A wrapper class over the TPathsUpdate thrift generated class. Please see * {@link Updateable.Update} for more information http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java new file mode 100644 index 0000000..16a1604 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Timer; +import org.apache.sentry.hdfs.service.thrift.TPathChanges; +import org.apache.sentry.provider.db.service.persistent.PathsImage; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.util.Map; +import java.util.Set; + +/** + * PathImageRetriever obtains a complete snapshot of Hive Paths from a persistent + * storage and translate it into {@code PathsUpdate} that the consumers, such as + * HDFS NameNod, can understand. + */ +public class PathImageRetriever implements ImageRetriever<PathsUpdate> { + + private final SentryStore sentryStore; + + PathImageRetriever(SentryStore sentryStore) { + this.sentryStore = sentryStore; + } + + @Override + public PathsUpdate retrieveFullImage(long seqNum) throws Exception { + try (final Timer.Context timerContext = + SentryHdfsMetricsUtil.getRetrievePathFullImageTimer.time()) { + + // Reads a up-to-date complete snapshot of Hive paths from the + // persistent storage, along with the sequence number of latest + // delta change the snapshot corresponds to. + PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); + long curSeqNum = pathsImage.getCurSeqNum(); + Map<String, Set<String>> pathImage = pathsImage.getPathImage(); + + // Translates the complete Hive paths snapshot into a PathsUpdate. + // Adds all <hiveObj, paths> mapping to be included in this paths update. + // And label it with the latest delta change sequence number for consumer + // to be aware of the next delta change it should continue with. + // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1613 + PathsUpdate pathsUpdate = new PathsUpdate(seqNum, true); + for (Map.Entry<String, Set<String>> pathEnt : pathImage.entrySet()) { + TPathChanges pathChange = pathsUpdate.newPathChange(pathEnt.getKey()); + + for (String path : pathEnt.getValue()) { + pathChange.addToAddPaths(PathsUpdate.splitPath(path)); + } + } + + SentryHdfsMetricsUtil.getPathChangesHistogram.update(pathsUpdate + .getPathChanges().size()); + return pathsUpdate; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java new file mode 100644 index 0000000..3017c9e --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Timer; +import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.sentry.hdfs.service.thrift.TRoleChanges; +import org.apache.sentry.provider.db.service.persistent.PermissionsImage; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * PermImageRetriever obtains a complete snapshot of Sentry permission from a persistent + * storage and translate it into {@code PermissionsUpdate} that the consumers, such as + * HDFS NameNod, can understand. + */ +public class PermImageRetriever implements ImageRetriever<PermissionsUpdate> { + + private final SentryStore sentryStore; + + PermImageRetriever(SentryStore sentryStore) { + this.sentryStore = sentryStore; + } + + @Override + public PermissionsUpdate retrieveFullImage(long seqNum) throws Exception { + try(Timer.Context timerContext = + SentryHdfsMetricsUtil.getRetrievePermFullImageTimer.time()) { + + // Read the most up-to-date snapshot of Sentry perm information, + // with a corresponding delta change sequence number. + PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage(); + long curSeqNum = permImage.getCurSeqNum(); + Map<String, Map<String, String>> privilegeImage = + permImage.getPrivilegeImage(); + Map<String, List<String>> roleImage = + permImage.getRoleImage(); + + // Translates the complete Sentry permission snapshot into a PermissionsUpdate. + // Adds the <hiveObj, <role, privileges>> mapping and the <role, groups> mapping + // to be included in the permission update. + // And label it with the latest delta change sequence number for consumer + // to be aware of the next delta change it should continue with. + TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, curSeqNum, + new HashMap<String, TPrivilegeChanges>(), + new HashMap<String, TRoleChanges>()); + + for (Map.Entry<String, Map<String, String>> privEnt : privilegeImage.entrySet()) { + String authzObj = privEnt.getKey(); + Map<String,String> privs = privEnt.getValue(); + tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges( + authzObj, privs, new HashMap<String, String>())); + } + + for (Map.Entry<String, List<String>> privEnt : roleImage.entrySet()) { + String role = privEnt.getKey(); + List<String> groups = privEnt.getValue(); + tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, + new LinkedList<String>())); + } + + PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); + // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1567 + permissionsUpdate.setSeqNum(seqNum); + SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update( + tPermUpdate.getPrivilegeChangesSize()); + SentryHdfsMetricsUtil.getRoleChangesHistogram.update( + tPermUpdate.getRoleChangesSize()); + return permissionsUpdate; + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java index e68c708..be14569 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java @@ -60,19 +60,30 @@ public class SentryHdfsMetricsUtil { MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification", "path-changes-size")); - // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever - // The time used for each retrieveFullImage - public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image")); - // The size of privilege changes for each retrieveFullImage + // Metrics for retrievePermFullImage in PermImageRetriever + // The time used for each retrievePermFullImage + public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image")); + // The size of privilege changes for each retrievePermFullImage public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "privilege-changes-size")); - // The size of role changes for each retrieveFullImage call + // The size of role changes for each retrievePermFullImage call public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "role-changes-size")); + // Metrics for retrievePathFullImage in PathImageRetriever + // The time used for each retrievePathFullImage + public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image")); + + // The size of path changes for each retrievePathFullImage + public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image", + "path-changes-size")); + + // Metrics for notifySentry HMS update in MetaStorePlugin // The timer used for each notifySentry public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer( http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index f68f690..029f9d5 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -18,23 +18,17 @@ package org.apache.sentry.hdfs; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.codahale.metrics.Timer; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.core.common.utils.SigUtils; import org.apache.sentry.hdfs.ServiceConstants.ServerConfig; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; -import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; -import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest; @@ -43,7 +37,6 @@ import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleRevokePrivil import org.apache.sentry.provider.db.service.thrift.TDropPrivilegesRequest; import org.apache.sentry.provider.db.service.thrift.TDropSentryRoleRequest; import org.apache.sentry.provider.db.service.thrift.TRenamePrivilegesRequest; -import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; import org.apache.sentry.provider.db.service.thrift.TSentryGroup; import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; import org.apache.sentry.service.thrift.HMSFollower; @@ -122,48 +115,6 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen public static volatile SentryPlugin instance; - static class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> { - - private final SentryStore sentryStore; - - public PermImageRetriever(SentryStore sentryStore) { - this.sentryStore = sentryStore; - } - - @Override - public PermissionsUpdate retrieveFullImage(long currSeqNum) throws Exception { - try(Timer.Context timerContext = - SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time()) { - - SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time(); - Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage(); - Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage(); - - TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum, - new HashMap<String, TPrivilegeChanges>(), - new HashMap<String, TRoleChanges>()); - for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) { - String authzObj = privEnt.getKey(); - HashMap<String,String> privs = privEnt.getValue(); - tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges( - authzObj, privs, new HashMap<String, String>())); - } - for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) { - String role = privEnt.getKey(); - LinkedList<String> groups = privEnt.getValue(); - tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>())); - } - PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); - permissionsUpdate.setSeqNum(currSeqNum); - SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update( - tPermUpdate.getPrivilegeChangesSize()); - SentryHdfsMetricsUtil.getRoleChangesHistogram.update( - tPermUpdate.getRoleChangesSize()); - return permissionsUpdate; - } - } - } - private UpdateForwarder<PathsUpdate> pathsUpdater; private UpdateForwarder<PermissionsUpdate> permsUpdater; // TODO: Each perm change sequence number should be generated during persistence at sentry store. @@ -202,10 +153,10 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen permImageRetriever = new PermImageRetriever(sentryStore); pathsUpdater = UpdateForwarder.create(conf, new UpdateableAuthzPaths( - pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs); + pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs, false); permsUpdater = UpdateForwarder.create(conf, new UpdateablePermissions(permImageRetriever), new PermissionsUpdate(0, false), - permImageRetriever, 100, initUpdateRetryDelayMs); + permImageRetriever, 100, initUpdateRetryDelayMs, true); LOGGER.info("Sentry HDFS plugin initialized !!"); instance = this; http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java index 6d5c607..22c5769 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -37,12 +37,6 @@ import org.slf4j.LoggerFactory; public class UpdateForwarder<K extends Updateable.Update> implements Updateable<K>, Closeable { - interface ExternalImageRetriever<K> { - - K retrieveFullImage(long currSeqNum) throws Exception; - - } - private final AtomicLong lastSeenSeqNum = new AtomicLong(0); protected final AtomicLong lastCommittedSeqNum = new AtomicLong(0); // Updates should be handled in order @@ -60,7 +54,7 @@ public class UpdateForwarder<K extends Updateable.Update> implements // UpdateLog is disabled when getMaxUpdateLogSize() = 0; private final int maxUpdateLogSize; - private final ExternalImageRetriever<K> imageRetreiver; + private final ImageRetriever<K> imageRetreiver; private volatile Updateable<K> updateable; @@ -72,16 +66,16 @@ public class UpdateForwarder<K extends Updateable.Update> implements private static final String UPDATABLE_TYPE_NAME = "update_forwarder"; public UpdateForwarder(Configuration conf, Updateable<K> updateable, - ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize) { - this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY); + ImageRetriever<K> imageRetreiver, int maxUpdateLogSize, boolean shouldInit) { + this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY, shouldInit); } protected UpdateForwarder(Configuration conf, Updateable<K> updateable, //NOPMD - ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize, - int initUpdateRetryDelay) { + ImageRetriever<K> imageRetreiver, int maxUpdateLogSize, + int initUpdateRetryDelay, boolean shouldInit) { this.maxUpdateLogSize = maxUpdateLogSize; this.imageRetreiver = imageRetreiver; - if (imageRetreiver != null) { + if (shouldInit) { spawnInitialUpdater(updateable, initUpdateRetryDelay); } else { this.updateable = updateable; @@ -89,17 +83,17 @@ public class UpdateForwarder<K extends Updateable.Update> implements } public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf, - Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver, - int maxUpdateLogSize) throws SentryPluginException { + Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver, + int maxUpdateLogSize, boolean shouldInit) throws SentryPluginException { return create(conf, updateable, update, imageRetreiver, maxUpdateLogSize, - INIT_UPDATE_RETRY_DELAY); + INIT_UPDATE_RETRY_DELAY, shouldInit); } public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf, - Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver, - int maxUpdateLogSize, int initUpdateRetryDelay) throws SentryPluginException { + Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver, + int maxUpdateLogSize, int initUpdateRetryDelay, boolean shouldInit) throws SentryPluginException { return new UpdateForwarder<K>(conf, updateable, imageRetreiver, - maxUpdateLogSize, initUpdateRetryDelay); + maxUpdateLogSize, initUpdateRetryDelay, shouldInit); } private void spawnInitialUpdater(final Updateable<K> updateable, http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java index fe2baa6..03c67d6 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java @@ -20,16 +20,14 @@ package org.apache.sentry.hdfs; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; - public class UpdateablePermissions implements Updateable<PermissionsUpdate>{ private static final String UPDATABLE_TYPE_NAME = "perm_update"; private AtomicLong seqNum = new AtomicLong(); - private final ExternalImageRetriever<PermissionsUpdate> imageRetreiver; + private final ImageRetriever<PermissionsUpdate> imageRetreiver; public UpdateablePermissions( - ExternalImageRetriever<PermissionsUpdate> imageRetreiver) { + ImageRetriever<PermissionsUpdate> imageRetreiver) { this.imageRetreiver = imageRetreiver; } @@ -62,5 +60,4 @@ public class UpdateablePermissions implements Updateable<PermissionsUpdate>{ public String getUpdateableTypeName() { return UPDATABLE_TYPE_NAME; } - } http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java index 0f0d0a7..d12b134 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java @@ -27,7 +27,6 @@ import org.apache.thrift.TException; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; import org.apache.sentry.hdfs.Updateable.Update; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.After; @@ -134,7 +133,7 @@ public class TestUpdateForwarder { } } - static class DummyImageRetreiver implements ExternalImageRetriever<DummyUpdate> { + static class DummyImageRetreiver implements ImageRetriever<DummyUpdate> { private String state; public void setState(String state) { @@ -164,7 +163,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10, true); Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum()); List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); Assert.assertTrue(allUpdates.size() == 1); @@ -184,7 +183,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -205,7 +204,7 @@ public class TestUpdateForwarder { Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED, false)); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), null, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), null, 5, false); updateForwarder.handleUpdateNotification(new DummyUpdate(-1, true).setState("a")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -232,7 +231,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -282,7 +281,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -328,7 +327,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java new file mode 100644 index 0000000..fd56ce2 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.persistent; + +import java.util.Map; +import java.util.Set; + +/** + * A container for complete hive paths snapshot. + * <p> + * It is composed by a hiveObj to Paths mapping and the sequence number/change ID + * of latest delta change that the snapshot maps to. + */ +public class PathsImage { + + // A full snapshot of hiveObj to Paths mapping. + private final Map<String, Set<String>> pathImage; + private final long curSeqNum; + + PathsImage(Map<String, Set<String>> pathImage, long curSeqNum) { + this.pathImage = pathImage; + this.curSeqNum = curSeqNum; + } + + public long getCurSeqNum() { + return curSeqNum; + } + + public Map<String, Set<String>> getPathImage() { + return pathImage; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java new file mode 100644 index 0000000..f03e93f --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.persistent; + +import java.util.List; +import java.util.Map; + +/** + * A container for complete sentry permission snapshot. + * <p> + * It is composed by a role to groups mapping, and hiveObj to < role, privileges > mapping. + * It also has the sequence number/change ID of latest delta change that the snapshot maps to. + */ +public class PermissionsImage { + + // A full snapshot of sentry role to groups mapping. + private final Map<String, List<String>> roleImage; + + // A full snapshot of hiveObj to <role, privileges> mapping. + private final Map<String, Map<String, String>> privilegeImage; + private final long curSeqNum; + + PermissionsImage(Map<String, List<String>> roleImage, + Map<String, Map<String, String>> privilegeImage, long curSeqNum) { + this.roleImage = roleImage; + this.privilegeImage = privilegeImage; + this.curSeqNum = curSeqNum; + } + + public long getCurSeqNum() { + return curSeqNum; + } + + public Map<String, Map<String, String>> getPrivilegeImage() { + return privilegeImage; + } + + public Map<String, List<String>> getRoleImage() { + return roleImage; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index a5a835e..6ea6d3f 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -2260,107 +2260,158 @@ public class SentryStore { ServerConfig.ADMIN_GROUPS, new String[]{})); } - public Map<String, HashMap<String, String>> retrieveFullPrivilegeImage() throws Exception { + /** + * Retrieves an up-to-date sentry permission snapshot. + * <p> + * It reads hiveObj to < role, privileges > mapping from {@link MSentryPrivilege} + * table and role to groups mapping from {@link MSentryGroup}. + * It also gets the changeID of latest delta update, from {@link MSentryPathChange}, that + * the snapshot corresponds to. + * + * @return a {@link PathsImage} contains the mapping of hiveObj to + * < role, privileges > and the mapping of role to < Groups >. + * For empty image returns {@link #EMPTY_CHANGE_ID} and empty maps. + * @throws Exception + */ + public PermissionsImage retrieveFullPermssionsImage() throws Exception { return tm.executeTransaction( - new TransactionBlock<Map<String, HashMap<String, String>>>() { - public Map<String, HashMap<String, String>> execute(PersistenceManager pm) - throws Exception { - Map<String, HashMap<String, String>> retVal = new HashMap<>(); - Query query = pm.newQuery(MSentryPrivilege.class); - QueryParamBuilder paramBuilder = newQueryParamBuilder(); - paramBuilder - .addNotNull(SERVER_NAME) - .addNotNull(DB_NAME) - .addNull(URI); + new TransactionBlock<PermissionsImage>() { + public PermissionsImage execute(PersistenceManager pm) + throws Exception { + // curChangeID could be 0, if Sentry server has been running before + // enable SentryPlugin(HDFS Sync feature). + long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPermChange.class); + Map<String, List<String>> roleImage = retrieveFullRoleImageCore(pm); + Map<String, Map<String, String>> privilegeMap = retrieveFullPrivilegeImageCore(pm); - query.setFilter(paramBuilder.toString()); - query.setOrdering("serverName ascending, dbName ascending, tableName ascending"); - @SuppressWarnings("unchecked") - List<MSentryPrivilege> privileges = - (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments()); - for (MSentryPrivilege mPriv : privileges) { - String authzObj = mPriv.getDbName(); - if (!isNULL(mPriv.getTableName())) { - authzObj = authzObj + "." + mPriv.getTableName(); - } - HashMap<String, String> pUpdate = retVal.get(authzObj); - if (pUpdate == null) { - pUpdate = new HashMap<>(); - retVal.put(authzObj, pUpdate); - } - for (MSentryRole mRole : mPriv.getRoles()) { - String existingPriv = pUpdate.get(mRole.getRoleName()); - if (existingPriv == null) { - pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase()); - } else { - pUpdate.put(mRole.getRoleName(), existingPriv + "," - + mPriv.getAction().toUpperCase()); - } - } - } - return retVal; + return new PermissionsImage(roleImage, privilegeMap, curChangeID); + } + }); + } + + /** + * Retrieves an up-to-date sentry privileges snapshot from {@code MSentryPrivilege} table. + * The snapshot is represented by mapping of hiveObj to role privileges. + * + * @param pm PersistenceManager + * @return a mapping of hiveObj to < role, privileges > + * @throws Exception + */ + private Map<String, Map<String, String>> retrieveFullPrivilegeImageCore(PersistenceManager pm) + throws Exception { + + Map<String, Map<String, String>> retVal = new HashMap<>(); + Query query = pm.newQuery(MSentryPrivilege.class); + QueryParamBuilder paramBuilder = newQueryParamBuilder(); + paramBuilder.addNotNull(SERVER_NAME) + .addNotNull(DB_NAME) + .addNull(URI); + + query.setFilter(paramBuilder.toString()); + query.setOrdering("serverName ascending, dbName ascending, tableName ascending"); + @SuppressWarnings("unchecked") + List<MSentryPrivilege> privileges = + (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments()); + for (MSentryPrivilege mPriv : privileges) { + String authzObj = mPriv.getDbName(); + if (!isNULL(mPriv.getTableName())) { + authzObj = authzObj + "." + mPriv.getTableName(); + } + Map<String, String> pUpdate = retVal.get(authzObj); + if (pUpdate == null) { + pUpdate = new HashMap<>(); + retVal.put(authzObj, pUpdate); + } + for (MSentryRole mRole : mPriv.getRoles()) { + String existingPriv = pUpdate.get(mRole.getRoleName()); + if (existingPriv == null) { + pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase()); + } else { + pUpdate.put(mRole.getRoleName(), existingPriv + "," + mPriv.getAction().toUpperCase()); } - }); + } + } + return retVal; } /** - * @return Mapping of Role -> [Groups] + * Retrieves an up-to-date sentry role snapshot from {@code MSentryGroup} table. + * The snapshot is represented by a role to groups map. + * + * @param pm PersistenceManager + * @return a mapping of Role to < Groups > + * @throws Exception */ - public Map<String, LinkedList<String>> retrieveFullRoleImage() throws Exception { - return tm.executeTransaction( - new TransactionBlock<Map<String, LinkedList<String>>>() { - public Map<String, LinkedList<String>> execute(PersistenceManager pm) throws Exception { - Query query = pm.newQuery(MSentryGroup.class); - @SuppressWarnings("unchecked") - List<MSentryGroup> groups = (List<MSentryGroup>) query.execute(); - if (groups.isEmpty()) { - return Collections.emptyMap(); - } + private Map<String, List<String>> retrieveFullRoleImageCore(PersistenceManager pm) + throws Exception { + Query query = pm.newQuery(MSentryGroup.class); + @SuppressWarnings("unchecked") + List<MSentryGroup> groups = (List<MSentryGroup>) query.execute(); + if (groups.isEmpty()) { + return Collections.emptyMap(); + } - Map<String, LinkedList<String>> retVal = new HashMap<>(); - for (MSentryGroup mGroup : groups) { - for (MSentryRole role : mGroup.getRoles()) { - LinkedList<String> rUpdate = retVal.get(role.getRoleName()); - if (rUpdate == null) { - rUpdate = new LinkedList<String>(); - retVal.put(role.getRoleName(), rUpdate); - } - rUpdate.add(mGroup.getGroupName()); - } - } - return retVal; - } - }); + Map<String, List<String>> retVal = new HashMap<>(); + for (MSentryGroup mGroup : groups) { + for (MSentryRole role : mGroup.getRoles()) { + List<String> rUpdate = retVal.get(role.getRoleName()); + if (rUpdate == null) { + rUpdate = new LinkedList<>(); + retVal.put(role.getRoleName(), rUpdate); + } + rUpdate.add(mGroup.getGroupName()); + } + } + return retVal; + } + + /** + * Retrieves an up-to-date hive paths snapshot. + * <p> + * It reads hiveObj to paths mapping from {@link MAuthzPathsMapping} table and + * gets the changeID of latest delta update, from {@link MSentryPathChange}, that + * the snapshot corresponds to. + * + * @return an up-to-date hive paths snapshot contains mapping of hiveObj to < Paths >. + * For empty image return {@link #EMPTY_CHANGE_ID} and a empty map. + * @throws Exception + */ + public PathsImage retrieveFullPathsImage() throws Exception { + return (PathsImage) tm.executeTransaction( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + // curChangeID could be 0 for the first full snapshot fetching + // from HMS. It does not have corresponding delta update. + long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPathChange.class); + Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm); + + return new PathsImage(pathImage, curChangeID); + } + }); } /** - * This returns a Mapping of Authz -> [Paths] + * Retrieves an up-to-date hive paths snapshot from {@code MAuthzPathsMapping} table. + * The snapshot is represented by a hiveObj to paths map. + * + * @return a mapping of hiveObj to < Paths >. */ - public Map<String, Set<String>> retrieveFullPathsImage() { - Map<String, Set<String>> result = new HashMap<>(); - try { - result = (Map<String, Set<String>>) tm.executeTransaction( - new TransactionBlock<Object>() { - public Object execute(PersistenceManager pm) throws Exception { - Map<String, Set<String>> retVal = new HashMap<>(); - Query query = pm.newQuery(MAuthzPathsMapping.class); - List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>) query.execute(); - for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) { - retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths()); - } - return retVal; - } - }); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); + private Map<String, Set<String>> retrieveFullPathsImageCore(PersistenceManager pm) { + Map<String, Set<String>> retVal = new HashMap<>(); + Query query = pm.newQuery(MAuthzPathsMapping.class); + Iterable<MAuthzPathsMapping> authzToPathsMappings = + (Iterable<MAuthzPathsMapping>) query.execute(); + + for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) { + retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths()); } - return result; + return retVal; } /** - * Persist a full hive snapshot into Sentry DB in a single transaction. + * Persist an up-to-date hive snapshot into Sentry DB in a single transaction. * - * @param authzPaths Mapping of hiveObj -> [Paths] + * @param authzPaths Mapping of hiveObj to < Paths < * @throws Exception */ public void persistFullPathsImage(final Map<String, Set<String>> authzPaths) throws Exception { http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index 91f15c0..75f855c 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -2188,21 +2188,67 @@ public class TestSentryStore extends org.junit.Assert { } @Test + public void testRetrieveFullPermssionsImage() throws Exception { + + // Create roles + String roleName1 = "privs-r1", roleName2 = "privs-r2"; + String groupName1 = "privs-g1"; + String grantor = "g1"; + sentryStore.createSentryRole(roleName1); + sentryStore.createSentryRole(roleName2); + + // Grant Privileges to the roles + TSentryPrivilege privilege1 = new TSentryPrivilege(); + privilege1.setPrivilegeScope("TABLE"); + privilege1.setServerName("server1"); + privilege1.setDbName("db1"); + privilege1.setTableName("tbl1"); + privilege1.setAction("SELECT"); + privilege1.setCreateTime(System.currentTimeMillis()); + sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege1); + sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName2, privilege1); + + TSentryPrivilege privilege2 = new TSentryPrivilege(); + privilege2.setPrivilegeScope("SERVER"); + privilege2.setServerName("server1"); + privilege1.setDbName("db2"); + privilege1.setAction("ALL"); + privilege2.setCreateTime(System.currentTimeMillis()); + sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName2, privilege2); + + // Grant roles to the groups + Set<TSentryGroup> groups = Sets.newHashSet(); + TSentryGroup group = new TSentryGroup(); + group.setGroupName(groupName1); + groups.add(group); + sentryStore.alterSentryRoleAddGroups(grantor, roleName1, groups); + sentryStore.alterSentryRoleAddGroups(grantor, roleName2, groups); + + PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage(); + Map<String, Map<String, String>> privs = permImage.getPrivilegeImage(); + Map<String, List<String>> roles = permImage.getRoleImage(); + assertEquals(2, privs.get("db1.tbl1").size()); + assertEquals(2, roles.size()); + } + + @Test public void testAuthzPathsMapping() throws Exception { sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1")); sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2")); - Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage(); - assertEquals(2, pathsImage.size()); - assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1")); + PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); + Map<String, Set<String>> pathImage = pathsImage.getPathImage(); + assertEquals(2, pathImage.size()); + assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1")); Map<String, Set<String>> authzPaths = new HashMap<>(); authzPaths.put("db2.table1", Sets.newHashSet("/user/hive/warehouse/db2.db/table1")); authzPaths.put("db2.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2")); sentryStore.persistFullPathsImage(authzPaths); pathsImage = sentryStore.retrieveFullPathsImage(); - assertEquals(4, pathsImage.size()); - assertEquals(Sets.newHashSet("/user/hive/warehouse/db2.db/table1"), pathsImage.get("db2.table1")); + pathImage = pathsImage.getPathImage(); + assertEquals(4, pathImage.size()); + assertEquals(Sets.newHashSet("/user/hive/warehouse/db2.db/table1"), pathImage.get("db2.table1")); } public void testQueryParamBuilder() {
