SENTRY-1566 Change-Id: I19936d9b3a5ead7dd063ecedad35e1ece8e311aa
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/dbd58709 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/dbd58709 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/dbd58709 Branch: refs/heads/sentry-ha-redesign-1 Commit: dbd5870901b933b9af3797765bef4fb8165b6b46 Parents: 81facc6 Author: hahao <[email protected]> Authored: Thu Jan 26 17:40:39 2017 -0800 Committer: hahao <[email protected]> Committed: Thu Jan 26 17:40:39 2017 -0800 ---------------------------------------------------------------------- .../org/apache/sentry/hdfs/ImageRetriever.java | 34 ++++ .../org/apache/sentry/hdfs/PathsUpdate.java | 22 ++ .../apache/sentry/hdfs/PathImageRetriever.java | 65 ++++++ .../apache/sentry/hdfs/PermImageRetriever.java | 84 ++++++++ .../sentry/hdfs/SentryHdfsMetricsUtil.java | 27 ++- .../org/apache/sentry/hdfs/SentryPlugin.java | 51 +---- .../org/apache/sentry/hdfs/UpdateForwarder.java | 30 ++- .../sentry/hdfs/UpdateablePermissions.java | 7 +- .../apache/sentry/hdfs/TestUpdateForwarder.java | 15 +- .../db/service/persistent/PathsImage.java | 40 ++++ .../db/service/persistent/PermissionsImage.java | 48 +++++ .../db/service/persistent/SentryStore.java | 204 ++++++++++++------- .../db/service/persistent/TestSentryStore.java | 7 +- 13 files changed, 466 insertions(+), 168 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java new file mode 100644 index 0000000..1147c07 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +/** + * Interface class for generating/retrieving a full image. + */ +public interface ImageRetriever<K> { + + /** + * Retrieve a full image of type k. + * + * @param seqNum the given seq number + * @return a full snapshot of type K + * @throws Exception + */ + K retrieveFullImage(long seqNum) throws Exception; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index ffb0756..a186198 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -24,6 +24,8 @@ import java.util.LinkedList; import java.util.List; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import org.apache.sentry.hdfs.service.thrift.TPathChanges; import org.apache.sentry.hdfs.service.thrift.TPathsUpdate; import org.apache.commons.httpclient.util.URIUtil; @@ -153,6 +155,26 @@ public class PathsUpdate implements Updateable.Update { } } + /** + * Get a path string concatenated by "/". + * + * @param paths + * @return + */ + public static String getPath(List<String> paths) { + return Joiner.on("/").join(paths); + } + + /** + * Split a path into a list containing the path tree using "/". + * + * @param path + * @return + */ + public static List<String> splitPath(String path) { + return Lists.newArrayList(Splitter.on("/").split(path)); + } + @Override public byte[] serialize() throws IOException { return ThriftSerializer.serialize(tPathsUpdate); http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java new file mode 100644 index 0000000..6cfd6d5 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Timer; +import org.apache.sentry.hdfs.service.thrift.TPathChanges; +import org.apache.sentry.provider.db.service.persistent.PathsImage; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class PathImageRetriever implements ImageRetriever<PathsUpdate> { + + private final SentryStore sentryStore; + + public PathImageRetriever(SentryStore sentryStore) { + this.sentryStore = sentryStore; + } + + @Override + public PathsUpdate retrieveFullImage(long seqNum) throws Exception { + try (final Timer.Context timerContext = + SentryHdfsMetricsUtil.getRetrievePathFullImageTimer.time()) { + + SentryHdfsMetricsUtil.getRetrievePathFullImageTimer.time(); + // Read the full paths snapshot from Sentry DB which + // associates with a up-to-date/corresponding sequence number. + PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); + long curSeqNum = pathsImage.getCurSeqNum(); + Map<String, Set<String>> pathImage = pathsImage.getPathImage(); + + // Generate a corresponding PathsUpdate. + // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1567 + PathsUpdate pathsUpdate = new PathsUpdate(seqNum, true); + for (Map.Entry<String, Set<String>> pathEnt : pathImage.entrySet()) { + TPathChanges pathChange = pathsUpdate.newPathChange(pathEnt.getKey()); + + for (String path : pathEnt.getValue()) { + pathChange.addToAddPaths(PathsUpdate.splitPath(path)); + } + } + + SentryHdfsMetricsUtil.getPathChangesHistogram.update(pathsUpdate + .getPathChanges().size()); + return pathsUpdate; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java new file mode 100644 index 0000000..56985c2 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Timer; +import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.sentry.hdfs.service.thrift.TRoleChanges; +import org.apache.sentry.provider.db.service.persistent.PermissionsImage; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +public class PermImageRetriever implements ImageRetriever<PermissionsUpdate> { + + private final SentryStore sentryStore; + + public PermImageRetriever(SentryStore sentryStore) { + this.sentryStore = sentryStore; + } + + @Override + public PermissionsUpdate retrieveFullImage(long seqNum) throws Exception { + try(Timer.Context timerContext = + SentryHdfsMetricsUtil.getRetrievePermFullImageTimer.time()) { + + SentryHdfsMetricsUtil.getRetrievePermFullImageTimer.time(); + + // Read the full permission snapshot from Sentry DB which + // associates with a up-to-date/corresponding sequence number. + PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage(); + long curSeqNum = permImage.getCurSeqNum(); + Map<String, HashMap<String, String>> privilegeImage = + permImage.getPrivilegeImage(); + Map<String, LinkedList<String>> roleImage = + permImage.getRoleImage(); + + // Generate a corresponding PermissionsUpdate. + TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, curSeqNum, + new HashMap<String, TPrivilegeChanges>(), + new HashMap<String, TRoleChanges>()); + + for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) { + String authzObj = privEnt.getKey(); + HashMap<String,String> privs = privEnt.getValue(); + tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges( + authzObj, privs, new HashMap<String, String>())); + } + + for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) { + String role = privEnt.getKey(); + LinkedList<String> groups = privEnt.getValue(); + tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>())); + } + + PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); + // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1567 + permissionsUpdate.setSeqNum(seqNum); + SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update( + tPermUpdate.getPrivilegeChangesSize()); + SentryHdfsMetricsUtil.getRoleChangesHistogram.update( + tPermUpdate.getRoleChangesSize()); + return permissionsUpdate; + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java index e68c708..be14569 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java @@ -60,19 +60,30 @@ public class SentryHdfsMetricsUtil { MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification", "path-changes-size")); - // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever - // The time used for each retrieveFullImage - public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image")); - // The size of privilege changes for each retrieveFullImage + // Metrics for retrievePermFullImage in PermImageRetriever + // The time used for each retrievePermFullImage + public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image")); + // The size of privilege changes for each retrievePermFullImage public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "privilege-changes-size")); - // The size of role changes for each retrieveFullImage call + // The size of role changes for each retrievePermFullImage call public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "role-changes-size")); + // Metrics for retrievePathFullImage in PathImageRetriever + // The time used for each retrievePathFullImage + public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image")); + + // The size of path changes for each retrievePathFullImage + public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image", + "path-changes-size")); + + // Metrics for notifySentry HMS update in MetaStorePlugin // The timer used for each notifySentry public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer( http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index 4c3e9d6..b99013e 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -18,19 +18,14 @@ package org.apache.sentry.hdfs; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.codahale.metrics.Timer; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.core.common.utils.SigUtils; import org.apache.sentry.hdfs.ServiceConstants.ServerConfig; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; -import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; @@ -119,48 +114,6 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen public static volatile SentryPlugin instance; - static class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> { - - private final SentryStore sentryStore; - - public PermImageRetriever(SentryStore sentryStore) { - this.sentryStore = sentryStore; - } - - @Override - public PermissionsUpdate retrieveFullImage(long currSeqNum) throws Exception { - try(Timer.Context timerContext = - SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time()) { - - SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time(); - Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage(); - Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage(); - - TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum, - new HashMap<String, TPrivilegeChanges>(), - new HashMap<String, TRoleChanges>()); - for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) { - String authzObj = privEnt.getKey(); - HashMap<String,String> privs = privEnt.getValue(); - tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges( - authzObj, privs, new HashMap<String, String>())); - } - for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) { - String role = privEnt.getKey(); - LinkedList<String> groups = privEnt.getValue(); - tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>())); - } - PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); - permissionsUpdate.setSeqNum(currSeqNum); - SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update( - tPermUpdate.getPrivilegeChangesSize()); - SentryHdfsMetricsUtil.getRoleChangesHistogram.update( - tPermUpdate.getRoleChangesSize()); - return permissionsUpdate; - } - } - } - private UpdateForwarder<PathsUpdate> pathsUpdater; private UpdateForwarder<PermissionsUpdate> permsUpdater; // TODO: Each perm change sequence number should be generated during persistence at sentry store. @@ -199,10 +152,10 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen permImageRetriever = new PermImageRetriever(sentryStore); pathsUpdater = UpdateForwarder.create(conf, new UpdateableAuthzPaths( - pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs); + pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs, false); permsUpdater = UpdateForwarder.create(conf, new UpdateablePermissions(permImageRetriever), new PermissionsUpdate(0, false), - permImageRetriever, 100, initUpdateRetryDelayMs); + permImageRetriever, 100, initUpdateRetryDelayMs, true); LOGGER.info("Sentry HDFS plugin initialized !!"); instance = this; http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java index 6d5c607..4bfc473 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java @@ -37,12 +37,6 @@ import org.slf4j.LoggerFactory; public class UpdateForwarder<K extends Updateable.Update> implements Updateable<K>, Closeable { - interface ExternalImageRetriever<K> { - - K retrieveFullImage(long currSeqNum) throws Exception; - - } - private final AtomicLong lastSeenSeqNum = new AtomicLong(0); protected final AtomicLong lastCommittedSeqNum = new AtomicLong(0); // Updates should be handled in order @@ -60,7 +54,7 @@ public class UpdateForwarder<K extends Updateable.Update> implements // UpdateLog is disabled when getMaxUpdateLogSize() = 0; private final int maxUpdateLogSize; - private final ExternalImageRetriever<K> imageRetreiver; + private final ImageRetriever<K> imageRetreiver; private volatile Updateable<K> updateable; @@ -72,16 +66,16 @@ public class UpdateForwarder<K extends Updateable.Update> implements private static final String UPDATABLE_TYPE_NAME = "update_forwarder"; public UpdateForwarder(Configuration conf, Updateable<K> updateable, - ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize) { - this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY); + ImageRetriever<K> imageRetreiver, int maxUpdateLogSize, boolean shouldInit) { + this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY, shouldInit); } protected UpdateForwarder(Configuration conf, Updateable<K> updateable, //NOPMD - ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize, - int initUpdateRetryDelay) { + ImageRetriever<K> imageRetreiver, int maxUpdateLogSize, + int initUpdateRetryDelay, boolean shouldInit) { this.maxUpdateLogSize = maxUpdateLogSize; this.imageRetreiver = imageRetreiver; - if (imageRetreiver != null) { + if (shouldInit) { spawnInitialUpdater(updateable, initUpdateRetryDelay); } else { this.updateable = updateable; @@ -89,17 +83,17 @@ public class UpdateForwarder<K extends Updateable.Update> implements } public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf, - Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver, - int maxUpdateLogSize) throws SentryPluginException { + Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver, + int maxUpdateLogSize, boolean shouldInit) throws SentryPluginException { return create(conf, updateable, update, imageRetreiver, maxUpdateLogSize, - INIT_UPDATE_RETRY_DELAY); + INIT_UPDATE_RETRY_DELAY, shouldInit); } public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf, - Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver, - int maxUpdateLogSize, int initUpdateRetryDelay) throws SentryPluginException { + Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver, + int maxUpdateLogSize, int initUpdateRetryDelay, boolean shouldInit) throws SentryPluginException { return new UpdateForwarder<K>(conf, updateable, imageRetreiver, - maxUpdateLogSize, initUpdateRetryDelay); + maxUpdateLogSize, initUpdateRetryDelay, shouldInit); } private void spawnInitialUpdater(final Updateable<K> updateable, http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java index fe2baa6..03c67d6 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java @@ -20,16 +20,14 @@ package org.apache.sentry.hdfs; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; - public class UpdateablePermissions implements Updateable<PermissionsUpdate>{ private static final String UPDATABLE_TYPE_NAME = "perm_update"; private AtomicLong seqNum = new AtomicLong(); - private final ExternalImageRetriever<PermissionsUpdate> imageRetreiver; + private final ImageRetriever<PermissionsUpdate> imageRetreiver; public UpdateablePermissions( - ExternalImageRetriever<PermissionsUpdate> imageRetreiver) { + ImageRetriever<PermissionsUpdate> imageRetreiver) { this.imageRetreiver = imageRetreiver; } @@ -62,5 +60,4 @@ public class UpdateablePermissions implements Updateable<PermissionsUpdate>{ public String getUpdateableTypeName() { return UPDATABLE_TYPE_NAME; } - } http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java index 0f0d0a7..d12b134 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java @@ -27,7 +27,6 @@ import org.apache.thrift.TException; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; import org.apache.sentry.hdfs.Updateable.Update; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.After; @@ -134,7 +133,7 @@ public class TestUpdateForwarder { } } - static class DummyImageRetreiver implements ExternalImageRetriever<DummyUpdate> { + static class DummyImageRetreiver implements ImageRetriever<DummyUpdate> { private String state; public void setState(String state) { @@ -164,7 +163,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10, true); Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum()); List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); Assert.assertTrue(allUpdates.size() == 1); @@ -184,7 +183,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -205,7 +204,7 @@ public class TestUpdateForwarder { Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED, false)); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), null, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), null, 5, false); updateForwarder.handleUpdateNotification(new DummyUpdate(-1, true).setState("a")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -232,7 +231,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -282,7 +281,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -328,7 +327,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java new file mode 100644 index 0000000..2aa837c --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.persistent; + +import java.util.Map; +import java.util.Set; + +public class PathsImage { + private final Map<String, Set<String>> pathImage; + private final long curSeqNum; + + public PathsImage(Map<String, Set<String>> pathImage, long curSeqNum) { + this.pathImage = pathImage; + this.curSeqNum = curSeqNum; + } + + public long getCurSeqNum() { + return curSeqNum; + } + + public Map<String, Set<String>> getPathImage() { + return pathImage; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java new file mode 100644 index 0000000..444f58f --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.persistent; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +public class PermissionsImage { + private Map<String, LinkedList<String>> roleImage; + private Map<String, HashMap<String, String>> privilegeImage; + private long curSeqNum; + + public PermissionsImage(Map<String, LinkedList<String>> roleImage, + Map<String, HashMap<String, String>> privilegeImage, long curSeqNum) { + this.roleImage = roleImage; + this.privilegeImage = privilegeImage; + this.curSeqNum = curSeqNum; + } + + public long getCurSeqNum() { + return curSeqNum; + } + + public Map<String, HashMap<String, String>> getPrivilegeImage() { + return privilegeImage; + } + + public Map<String, LinkedList<String>> getRoleImage() { + return roleImage; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 1bf4e82..46e23c6 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -49,6 +49,7 @@ import org.apache.sentry.core.common.exception.*; import org.apache.sentry.core.common.utils.SentryConstants; import org.apache.sentry.core.model.db.AccessConstants; import org.apache.sentry.core.model.db.DBModelAuthorizable.AuthorizableType; +import org.apache.sentry.hdfs.Updateable; import org.apache.sentry.provider.db.service.model.*; import org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor; import org.apache.sentry.provider.db.service.thrift.TSentryActiveRoleSet; @@ -2155,99 +2156,148 @@ public class SentryStore { ServerConfig.ADMIN_GROUPS, new String[]{})); } - public Map<String, HashMap<String, String>> retrieveFullPrivilegeImage() throws Exception { - Map<String, HashMap<String, String>> result = new HashMap<>(); + /** + * Retrieve an up-to-date snapshot of Permission image + * Internally calls retrieveFullPrivilegeImage and + * retrieveFullRoleImage. + * + * @return a Paths image contains mapping of Role -> [Privileges] + * and mapping of Role -> [Groups]. For empty image + * return EMPTY_CHANGE_ID and empty maps. + * @throws Exception + */ + public PermissionsImage retrieveFullPermssionsImage() throws Exception { return tm.executeTransaction( - new TransactionBlock<Map<String, HashMap<String, String>>>() { - public Map<String, HashMap<String, String>> execute(PersistenceManager pm) + new TransactionBlock<PermissionsImage>() { + public PermissionsImage execute(PersistenceManager pm) throws Exception { - Map<String, HashMap<String, String>> retVal = new HashMap<>(); - Query query = pm.newQuery(MSentryPrivilege.class); - QueryParamBuilder paramBuilder = new QueryParamBuilder(); - paramBuilder - .addNotNull(SERVER_NAME) - .addNotNull(DB_NAME) - .addNull(URI); - - query.setFilter(paramBuilder.toString()); - query.setOrdering("serverName ascending, dbName ascending, tableName ascending"); - @SuppressWarnings("unchecked") - List<MSentryPrivilege> privileges = - (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments()); - for (MSentryPrivilege mPriv : privileges) { - String authzObj = mPriv.getDbName(); - if (!isNULL(mPriv.getTableName())) { - authzObj = authzObj + "." + mPriv.getTableName(); - } - HashMap<String, String> pUpdate = retVal.get(authzObj); - if (pUpdate == null) { - pUpdate = new HashMap<String, String>(); - retVal.put(authzObj, pUpdate); - } - for (MSentryRole mRole : mPriv.getRoles()) { - String existingPriv = pUpdate.get(mRole.getRoleName()); - if (existingPriv == null) { - pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase()); - } else { - pUpdate.put(mRole.getRoleName(), existingPriv + "," - + mPriv.getAction().toUpperCase()); - } - } + long curChangeID = getLastProcessedPermChangeIDCore(pm); + Map<String, LinkedList<String>> roleImage = retrieveFullRoleImage(pm); + Map<String, HashMap<String, String>> privilegeMap = retrieveFullPrivilegeImage(pm); + + if (curChangeID == EMPTY_CHANGE_ID && (!roleImage.isEmpty() + || !privilegeMap.isEmpty())) { + throw new Exception("Non-empty full permission image should not have" + + "an empty change ID."); } - return retVal; + return new PermissionsImage(roleImage, privilegeMap, curChangeID); } }); } /** + * Retrieve an up-to-date snapshot of Role -> [Privileges] + * + * @param pm PersistenceManager + * @return Mapping of Role -> [Privileges] + * @throws Exception + */ + @SuppressWarnings("unchecked") + private Map<String, HashMap<String, String>> retrieveFullPrivilegeImage(PersistenceManager pm) + throws Exception { + Map<String, HashMap<String, String>> retVal = new HashMap<>(); + Query query = pm.newQuery(MSentryPrivilege.class); + QueryParamBuilder paramBuilder = new QueryParamBuilder(); + paramBuilder.addNotNull(SERVER_NAME).addNotNull(DB_NAME) + .addNull(URI); + + query.setFilter(paramBuilder.toString()); + query.setOrdering("serverName ascending, dbName ascending, tableName ascending"); + List<MSentryPrivilege> privileges = + (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments()); + if (privileges != null) { + for (MSentryPrivilege mPriv : privileges) { + String authzObj = mPriv.getDbName(); + if (!isNULL(mPriv.getTableName())) { + authzObj = authzObj + "." + mPriv.getTableName(); + } + HashMap<String, String> pUpdate = retVal.get(authzObj); + if (pUpdate == null) { + pUpdate = new HashMap<>(); + retVal.put(authzObj, pUpdate); + } + for (MSentryRole mRole : mPriv.getRoles()) { + String existingPriv = pUpdate.get(mRole.getRoleName()); + if (existingPriv == null) { + pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase()); + } else { + pUpdate.put(mRole.getRoleName(), existingPriv + "," + + mPriv.getAction().toUpperCase()); + } + } + } + } + return retVal; + } + + /** + * Retrieve an up-to-date snapshot of Role -> [Groups] + * + * @param pm PersistenceManager * @return Mapping of Role -> [Groups] + * @throws Exception */ - public Map<String, LinkedList<String>> retrieveFullRoleImage() throws Exception { - Map<String, LinkedList<String>> result = new HashMap<>(); - return tm.executeTransaction( - new TransactionBlock<Map<String, LinkedList<String>>>() { - public Map<String, LinkedList<String>> execute(PersistenceManager pm) throws Exception { - Map<String, LinkedList<String>> retVal = new HashMap<>(); - Query query = pm.newQuery(MSentryGroup.class); - @SuppressWarnings("unchecked") - List<MSentryGroup> groups = (List<MSentryGroup>) query.execute(); - for (MSentryGroup mGroup : groups) { - for (MSentryRole role : mGroup.getRoles()) { - LinkedList<String> rUpdate = retVal.get(role.getRoleName()); - if (rUpdate == null) { - rUpdate = new LinkedList<String>(); - retVal.put(role.getRoleName(), rUpdate); - } - rUpdate.add(mGroup.getGroupName()); - } - } - return retVal; + private Map<String, LinkedList<String>> retrieveFullRoleImage(PersistenceManager pm) throws Exception { + Map<String, LinkedList<String>> retVal = new HashMap<>(); + Query query = pm.newQuery(MSentryGroup.class); + List<MSentryGroup> groups = (List<MSentryGroup>) query.execute(); + + if (groups != null) { + for (MSentryGroup mGroup : groups) { + for (MSentryRole role : mGroup.getRoles()) { + LinkedList<String> rUpdate = retVal.get(role.getRoleName()); + if (rUpdate == null) { + rUpdate = new LinkedList<>(); + retVal.put(role.getRoleName(), rUpdate); } - }); + rUpdate.add(mGroup.getGroupName()); + } + } + } + + return retVal; } /** - * This returns a Mapping of Authz -> [Paths] + * Retrieve an up-to-date snapshot of hiveAuthz -> [Paths] + * Internally calls retrieveFullPathsImageCore + * + * @return a Paths image contains mapping of hiveObj -> [Paths]. + * For empty image return EMPTY_CHANGE_ID and a empty map. + * @throws Exception */ - public Map<String, Set<String>> retrieveFullPathsImage() { - Map<String, Set<String>> result = new HashMap<>(); - try { - result = (Map<String, Set<String>>) tm.executeTransaction( - new TransactionBlock() { - public Object execute(PersistenceManager pm) throws Exception { - Map<String, Set<String>> retVal = new HashMap<>(); - Query query = pm.newQuery(MAuthzPathsMapping.class); - List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>) query.execute(); - for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) { - retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths()); - } - return retVal; - } - }); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); + public PathsImage retrieveFullPathsImage() throws Exception { + return (PathsImage) tm.executeTransaction( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + long curChangeID = getLastProcessedPermChangeIDCore(pm); + Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm); + + if (curChangeID == EMPTY_CHANGE_ID && !pathImage.isEmpty()) { + throw new Exception("Non-empty full paths image should not have" + + "an empty change ID."); + } + return new PathsImage(pathImage, curChangeID); + } + }); + } + + /** + * Retrieve an up-to-date snapshot of hiveAuthz -> [Paths] + * + * @return Mapping of hiveObj -> [Paths] + */ + public Map<String, Set<String>> retrieveFullPathsImageCore(PersistenceManager pm) { + Map<String, Set<String>> retVal = new HashMap<>(); + Query query = pm.newQuery(MAuthzPathsMapping.class); + List<MAuthzPathsMapping> authzToPathsMappings = + (List<MAuthzPathsMapping>) query.execute(); + if (authzToPathsMappings != null) { + for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) { + retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths()); + } } - return result; + return retVal; } public void createAuthzPathsMapping(final String hiveObj, http://git-wip-us.apache.org/repos/asf/sentry/blob/dbd58709/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index 59c9567..a26f996 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -2194,9 +2194,10 @@ public class TestSentryStore extends org.junit.Assert { sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1")); sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2")); - Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage(); - assertEquals(2, pathsImage.size()); - assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1")); + PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); + Map<String, Set<String>> pathImage = pathsImage.getPathImage(); + assertEquals(2, pathImage.size()); + assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1")); } public void testQueryParamBuilder() {
