SENTRY-1511 Change-Id: I59f888e6a972f409950f8465f65f21293713127e
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/c22dbe45 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/c22dbe45 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/c22dbe45 Branch: refs/heads/sentry-ha-redesign_hdfs Commit: c22dbe459f1bdb4eef86db9bbdbb5b97b88cb6bc Parents: 0163bd9 Author: hahao <[email protected]> Authored: Tue Nov 29 18:34:45 2016 -0800 Committer: hahao <[email protected]> Committed: Tue Nov 29 18:34:45 2016 -0800 ---------------------------------------------------------------------- .../sentry/hdfs/ExternalImageRetriever.java | 25 ++++ .../org/apache/sentry/hdfs/PathsUpdate.java | 26 ++++ .../apache/sentry/hdfs/PermissionsUpdate.java | 16 +++ .../java/org/apache/sentry/hdfs/Updateable.java | 6 + .../org/apache/sentry/hdfs/TestPathsUpdate.java | 15 +++ .../sentry/hdfs/TestPermissionUpdate.java | 41 +++++++ .../sentry/hdfs/UpdateableAuthzPermissions.java | 1 + .../apache/sentry/hdfs/PathImageRetriever.java | 54 ++++++++ .../apache/sentry/hdfs/PermImageRetriever.java | 65 ++++++++++ .../sentry/hdfs/SentryHdfsMetricsUtil.java | 27 ++-- .../org/apache/sentry/hdfs/SentryPlugin.java | 123 ++++++++----------- .../org/apache/sentry/hdfs/UpdateForwarder.java | 22 ++-- .../sentry/hdfs/UpdateablePermissions.java | 3 - .../apache/sentry/hdfs/TestUpdateForwarder.java | 24 ++-- .../provider/db/SentryPolicyStorePlugin.java | 20 +-- .../db/service/model/MSentryPathChange.java | 99 +++++++++++++++ .../db/service/model/MSentryPermChange.java | 101 +++++++++++++++ .../provider/db/service/model/package.jdo | 28 +++++ .../db/service/persistent/SentryStore.java | 122 ++++++++++++++++-- .../thrift/SentryPolicyStoreProcessor.java | 15 ++- .../db/service/persistent/TestSentryStore.java | 40 +++++- 21 files changed, 744 insertions(+), 129 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ExternalImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ExternalImageRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ExternalImageRetriever.java new file mode 100644 index 0000000..5646358 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ExternalImageRetriever.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + + +public interface ExternalImageRetriever<K> { + + K retrieveFullImage(long currSeqNum); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index 7cfb3bf..9065fcf 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -24,7 +24,9 @@ import java.util.LinkedList; import java.util.List; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import org.apache.sentry.hdfs.service.thrift.TPathChanges; +import com.google.common.base.Splitter; import org.apache.sentry.hdfs.service.thrift.TPathsUpdate; import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.httpclient.URIException; @@ -33,6 +35,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.conf.Configuration; import com.google.common.collect.Lists; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TSimpleJSONProtocol; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,6 +158,14 @@ public class PathsUpdate implements Updateable.Update { } } + public static String getPath(List<String> path) { + return Joiner.on("/").join(path); + } + + public static List<String> splitPath(String path) { + return Lists.newArrayList(Splitter.on("/").split(path)); + } + @Override public byte[] serialize() throws IOException { return ThriftSerializer.serialize(tPathsUpdate); @@ -162,4 +176,16 @@ public class PathsUpdate implements Updateable.Update { ThriftSerializer.deserialize(tPathsUpdate, data); } + @Override + public void deserializeFromJSON(String update) throws TException { + TDeserializer deserializer = new TDeserializer(new TSimpleJSONProtocol.Factory()); + deserializer.fromString(tPathsUpdate, update); + } + + @Override + public String serializeToJSON() throws TException { + TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory()); + return serializer.toString(tPathsUpdate); + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java index 9834923..48d3cb8 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java @@ -25,6 +25,10 @@ import java.util.LinkedList; import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TSimpleJSONProtocol; public class PermissionsUpdate implements Updateable.Update { @@ -106,4 +110,16 @@ public class PermissionsUpdate implements Updateable.Update { public void deserialize(byte[] data) throws IOException { ThriftSerializer.deserialize(tPermUpdate, data); } + + @Override + public void deserializeFromJSON(String update) throws TException { + TDeserializer deserializer = new TDeserializer(new TSimpleJSONProtocol.Factory()); + deserializer.fromString(tPermUpdate, update); + } + + @Override + public String serializeToJSON() throws TException { + TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory()); + return serializer.toString(tPermUpdate); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java index 4dc3a0c..10d84d6 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java @@ -17,6 +17,8 @@ */ package org.apache.sentry.hdfs; +import org.apache.thrift.TException; + import java.io.IOException; import java.util.concurrent.locks.ReadWriteLock; @@ -39,6 +41,10 @@ public interface Updateable<K extends Updateable.Update> { byte[] serialize() throws IOException; void deserialize(byte data[]) throws IOException; + + void deserializeFromJSON(String update) throws TException; + + String serializeToJSON() throws TException; } /** http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPathsUpdate.java index 53243b4..eea1bda 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPathsUpdate.java @@ -19,6 +19,9 @@ package org.apache.sentry.hdfs; import java.util.List; +import org.apache.sentry.hdfs.service.thrift.TPathChanges; +import org.apache.sentry.hdfs.service.thrift.TPathsUpdate; +import org.apache.thrift.TException; import org.junit.Test; import org.junit.Assert; @@ -59,4 +62,16 @@ public class TestPathsUpdate { System.out.println(results); Assert.assertNull("Parse path without throwing exception",results); } + + @Test + public void testJson() throws SentryMalformedPathException, TException{ + PathsUpdate update = new PathsUpdate(1, true); + TPathChanges pathChange = update.newPathChange("db1.tbl12"); + pathChange.addToAddPaths(PathsUpdate.parsePath("hdfs:///db1/tbl12/part121")); + + // Serialize and deserialize the PermssionUpdate object should equals to the original one. + TPathsUpdate before = update.toThrift(); + update.deserializeFromJSON(update.serializeToJSON()); + junit.framework.Assert.assertEquals(before, update.toThrift()); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPermissionUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPermissionUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPermissionUpdate.java new file mode 100644 index 0000000..d0b492c --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/src/test/java/org/apache/sentry/hdfs/TestPermissionUpdate.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + + +import junit.framework.Assert; +import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.thrift.TException; +import org.junit.Test; + +public class TestPermissionUpdate { + + @Test + public void testSerializeDeserializeJSON() throws TException { + PermissionsUpdate update = new PermissionsUpdate(0, false); + TPrivilegeChanges privUpdate = update.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS); + privUpdate.putToAddPrivileges("newAuthz", "newAuthz"); + privUpdate.putToDelPrivileges("oldAuthz", "oldAuthz"); + + // Serialize and deserialize the PermssionUpdate object should equals to the original one. + TPermissionsUpdate before = update.toThrift(); + update.deserializeFromJSON(update.serializeToJSON()); + Assert.assertEquals(before, update.toThrift()); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java index 2472928..ee7c101 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java @@ -217,6 +217,7 @@ public class UpdateableAuthzPermissions implements AuthzPermissions, Updateable< @Override public PermissionsUpdate createFullImageUpdate(long currSeqNum) { + // In-memory cache PermissionsUpdate retVal = new PermissionsUpdate(currSeqNum, true); for (PrivilegeInfo pInfo : perms.getAllPrivileges()) { TPrivilegeChanges pUpdate = retVal.addPrivilegeUpdate(pInfo.getAuthzObj()); http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java new file mode 100644 index 0000000..8b3fd4e --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Timer; +import org.apache.sentry.hdfs.service.thrift.TPathChanges; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.util.Map; +import java.util.Set; + +public class PathImageRetriever implements ExternalImageRetriever<PathsUpdate> { + + private final SentryStore sentryStore; + + public PathImageRetriever(SentryStore sentryStore) { + this.sentryStore = sentryStore; + } + + @Override + public PathsUpdate retrieveFullImage(long currSeqNum) { + final Timer.Context timerContext = SentryHdfsMetricsUtil.getRetrievePathFullImageTimer.time(); + Map<String, Set<String>> pathImage = sentryStore.retrieveFullPathsImage(); + PathsUpdate pathsUpdate = new PathsUpdate(currSeqNum, true); + + for (Map.Entry<String, Set<String>> pathEnt : pathImage.entrySet()) { + TPathChanges pathChange = pathsUpdate.newPathChange(pathEnt.getKey()); + + for (String path : pathEnt.getValue()) { + pathChange.addToAddPaths(PathsUpdate.splitPath(path)); + } + } + + timerContext.stop(); + SentryHdfsMetricsUtil.getPathChangesHistogram.update(pathsUpdate.getPathChanges().size()); + + return pathsUpdate; + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java new file mode 100644 index 0000000..d71f4d3 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Timer; +import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.sentry.hdfs.service.thrift.TRoleChanges; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +public class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> { + + private final SentryStore sentryStore; + + public PermImageRetriever(SentryStore sentryStore) { + this.sentryStore = sentryStore; + } + + @Override + public PermissionsUpdate retrieveFullImage(long currSeqNum) { + final Timer.Context timerContext = SentryHdfsMetricsUtil.getRetrievePermFullImageTimer.time(); + Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage(); + Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage(); + + TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum, new HashMap<String, TPrivilegeChanges>(), new HashMap<String, TRoleChanges>()); + + for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) { + String authzObj = privEnt.getKey(); + HashMap<String, String> privs = privEnt.getValue(); + tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges(authzObj, privs, new HashMap<String, String>())); + } + + for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) { + String role = privEnt.getKey(); + LinkedList<String> groups = privEnt.getValue(); + tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>())); + } + + PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); + permissionsUpdate.setSeqNum(currSeqNum); + timerContext.stop(); + SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update(tPermUpdate.getPrivilegeChangesSize()); + SentryHdfsMetricsUtil.getRoleChangesHistogram.update(tPermUpdate.getRoleChangesSize()); + return permissionsUpdate; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java index e68c708..be14569 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java @@ -60,19 +60,30 @@ public class SentryHdfsMetricsUtil { MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification", "path-changes-size")); - // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever - // The time used for each retrieveFullImage - public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image")); - // The size of privilege changes for each retrieveFullImage + // Metrics for retrievePermFullImage in PermImageRetriever + // The time used for each retrievePermFullImage + public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image")); + // The size of privilege changes for each retrievePermFullImage public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "privilege-changes-size")); - // The size of role changes for each retrieveFullImage call + // The size of role changes for each retrievePermFullImage call public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( - MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "role-changes-size")); + // Metrics for retrievePathFullImage in PathImageRetriever + // The time used for each retrievePathFullImage + public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image")); + + // The size of path changes for each retrievePathFullImage + public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image", + "path-changes-size")); + + // Metrics for notifySentry HMS update in MetaStorePlugin // The timer used for each notifySentry public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer( http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index f3926a2..855ee10 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -18,17 +18,13 @@ package org.apache.sentry.hdfs; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -import com.codahale.metrics.Timer; +import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.hdfs.ServiceConstants.ServerConfig; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; -import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; @@ -52,47 +48,6 @@ public class SentryPlugin implements SentryPolicyStorePlugin { public static volatile SentryPlugin instance; - static class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> { - - private final SentryStore sentryStore; - - public PermImageRetriever(SentryStore sentryStore) { - this.sentryStore = sentryStore; - } - - @Override - public PermissionsUpdate retrieveFullImage(long currSeqNum) { - final Timer.Context timerContext = - SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time(); - Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage(); - Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage(); - - TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum, - new HashMap<String, TPrivilegeChanges>(), - new HashMap<String, TRoleChanges>()); - for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) { - String authzObj = privEnt.getKey(); - HashMap<String,String> privs = privEnt.getValue(); - tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges( - authzObj, privs, new HashMap<String, String>())); - } - for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) { - String role = privEnt.getKey(); - LinkedList<String> groups = privEnt.getValue(); - tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>())); - } - PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); - permissionsUpdate.setSeqNum(currSeqNum); - timerContext.stop(); - SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update( - tPermUpdate.getPrivilegeChangesSize()); - SentryHdfsMetricsUtil.getRoleChangesHistogram.update( - tPermUpdate.getRoleChangesSize()); - return permissionsUpdate; - } - - } - private UpdateForwarder<PathsUpdate> pathsUpdater; private UpdateForwarder<PermissionsUpdate> permsUpdater; private final AtomicLong permSeqNum = new AtomicLong(5); @@ -114,10 +69,10 @@ public class SentryPlugin implements SentryPolicyStorePlugin { permImageRetriever = new PermImageRetriever(sentryStore); pathsUpdater = UpdateForwarder.create(conf, new UpdateableAuthzPaths( - pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs); + pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs, false); permsUpdater = UpdateForwarder.create(conf, new UpdateablePermissions(permImageRetriever), new PermissionsUpdate(0, false), - permImageRetriever, 100, initUpdateRetryDelayMs); + permImageRetriever, 100, initUpdateRetryDelayMs, true); LOGGER.info("Sentry HDFS plugin initialized !!"); instance = this; } @@ -137,19 +92,20 @@ public class SentryPlugin implements SentryPolicyStorePlugin { } @Override - public void onAlterSentryRoleAddGroups( + public Updateable.Update onAlterSentryRoleAddGroups( TAlterSentryRoleAddGroupsRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); TRoleChanges rUpdate = update.addRoleUpdate(request.getRoleName()); for (TSentryGroup group : request.getGroups()) { rUpdate.addToAddGroups(group.getGroupName()); } - permsUpdater.handleUpdateNotification(update); + // permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); + return update; } @Override - public void onAlterSentryRoleDeleteGroups( + public Updateable.Update onAlterSentryRoleDeleteGroups( TAlterSentryRoleDeleteGroupsRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); @@ -157,38 +113,45 @@ public class SentryPlugin implements SentryPolicyStorePlugin { for (TSentryGroup group : request.getGroups()) { rUpdate.addToDelGroups(group.getGroupName()); } - permsUpdater.handleUpdateNotification(update); + // permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); + return update; } @Override - public void onAlterSentryRoleGrantPrivilege( - TAlterSentryRoleGrantPrivilegeRequest request) - throws SentryPluginException { + public void onAlterSentryRoleGrantPrivilege(TAlterSentryRoleGrantPrivilegeRequest request, + Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap) throws SentryPluginException { + if (privilegesUpdateMap == null) { + privilegesUpdateMap = Maps.newHashMap(); + } + if (request.isSetPrivileges()) { String roleName = request.getRoleName(); for (TSentryPrivilege privilege : request.getPrivileges()) { if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) { - onAlterSentryRoleGrantPrivilegeCore(roleName, privilege); + PermissionsUpdate update = onAlterSentryRoleGrantPrivilegeCore(roleName, privilege); + privilegesUpdateMap.put(privilege, update); } } } } - private void onAlterSentryRoleGrantPrivilegeCore(String roleName, TSentryPrivilege privilege) + private PermissionsUpdate onAlterSentryRoleGrantPrivilegeCore(String roleName, TSentryPrivilege privilege) throws SentryPluginException { String authzObj = getAuthzObj(privilege); if (authzObj != null) { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); update.addPrivilegeUpdate(authzObj).putToAddPrivileges( roleName, privilege.getAction().toUpperCase()); - permsUpdater.handleUpdateNotification(update); + //permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + "].."); + return update; } + return null; } @Override - public void onRenameSentryPrivilege(TRenamePrivilegesRequest request) + public Updateable.Update onRenameSentryPrivilege(TRenamePrivilegesRequest request) throws SentryPluginException { String oldAuthz = getAuthzObj(request.getOldAuthorizable()); String newAuthz = getAuthzObj(request.getNewAuthorizable()); @@ -196,64 +159,74 @@ public class SentryPlugin implements SentryPolicyStorePlugin { TPrivilegeChanges privUpdate = update.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS); privUpdate.putToAddPrivileges(newAuthz, newAuthz); privUpdate.putToDelPrivileges(oldAuthz, oldAuthz); - permsUpdater.handleUpdateNotification(update); + // permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + newAuthz + ", " + oldAuthz + "].."); + return update; } @Override - public void onAlterSentryRoleRevokePrivilege( + public Updateable.Update onAlterSentryRoleRevokePrivilege( TAlterSentryRoleRevokePrivilegeRequest request) throws SentryPluginException { + PermissionsUpdate update = null; + if (request.isSetPrivileges()) { String roleName = request.getRoleName(); for (TSentryPrivilege privilege : request.getPrivileges()) { if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) { - onAlterSentryRoleRevokePrivilegeCore(roleName, privilege); + update = onAlterSentryRoleRevokePrivilegeCore(roleName, privilege); } } } - } - public boolean isOutOfSync() { - return outOfSync; - } - - public void setOutOfSync(boolean outOfSync) { - this.outOfSync = outOfSync; + return update; } - private void onAlterSentryRoleRevokePrivilegeCore(String roleName, TSentryPrivilege privilege) + private PermissionsUpdate onAlterSentryRoleRevokePrivilegeCore(String roleName, TSentryPrivilege privilege) throws SentryPluginException { String authzObj = getAuthzObj(privilege); if (authzObj != null) { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); update.addPrivilegeUpdate(authzObj).putToDelPrivileges( roleName, privilege.getAction().toUpperCase()); - permsUpdater.handleUpdateNotification(update); + // permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); + return update; } + + return null; } @Override - public void onDropSentryRole(TDropSentryRoleRequest request) + public Updateable.Update onDropSentryRole(TDropSentryRoleRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); update.addPrivilegeUpdate(PermissionsUpdate.ALL_AUTHZ_OBJ).putToDelPrivileges( request.getRoleName(), PermissionsUpdate.ALL_AUTHZ_OBJ); update.addRoleUpdate(request.getRoleName()).addToDelGroups(PermissionsUpdate.ALL_GROUPS); - permsUpdater.handleUpdateNotification(update); + // permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); + return update; } @Override - public void onDropSentryPrivilege(TDropPrivilegesRequest request) + public Updateable.Update onDropSentryPrivilege(TDropPrivilegesRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); String authzObj = getAuthzObj(request.getAuthorizable()); update.addPrivilegeUpdate(authzObj).putToDelPrivileges( PermissionsUpdate.ALL_ROLES, PermissionsUpdate.ALL_ROLES); - permsUpdater.handleUpdateNotification(update); + // permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); + return update; + } + + public boolean isOutOfSync() { + return outOfSync; + } + + public void setOutOfSync(boolean outOfSync) { + this.outOfSync = outOfSync; } private String getAuthzObj(TSentryPrivilege privilege) { http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java index ea1c8f6..90c5f28 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java @@ -37,12 +37,6 @@ import org.slf4j.LoggerFactory; public class UpdateForwarder<K extends Updateable.Update> implements Updateable<K>, Closeable { - interface ExternalImageRetriever<K> { - - K retrieveFullImage(long currSeqNum); - - } - private final AtomicLong lastSeenSeqNum = new AtomicLong(0); protected final AtomicLong lastCommittedSeqNum = new AtomicLong(0); // Updates should be handled in order @@ -72,15 +66,15 @@ public class UpdateForwarder<K extends Updateable.Update> implements private static final String UPDATABLE_TYPE_NAME = "update_forwarder"; public UpdateForwarder(Configuration conf, Updateable<K> updateable, - ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize) { - this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY); + ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize, boolean shouldInit) { + this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY, shouldInit); } public UpdateForwarder(Configuration conf, Updateable<K> updateable, //NOPMD ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize, - int initUpdateRetryDelay) { + int initUpdateRetryDelay, boolean shouldInit) { this.maxUpdateLogSize = maxUpdateLogSize; this.imageRetreiver = imageRetreiver; - if (imageRetreiver != null) { + if (shouldInit) { spawnInitialUpdater(updateable, initUpdateRetryDelay); } else { this.updateable = updateable; @@ -89,16 +83,16 @@ public class UpdateForwarder<K extends Updateable.Update> implements public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf, Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver, - int maxUpdateLogSize) throws SentryPluginException { + int maxUpdateLogSize, boolean shouldInit) throws SentryPluginException { return create(conf, updateable, update, imageRetreiver, maxUpdateLogSize, - INIT_UPDATE_RETRY_DELAY); + INIT_UPDATE_RETRY_DELAY, shouldInit); } public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf, Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver, - int maxUpdateLogSize, int initUpdateRetryDelay) throws SentryPluginException { + int maxUpdateLogSize, int initUpdateRetryDelay, boolean shouldInit) throws SentryPluginException { return new UpdateForwarder<K>(conf, updateable, imageRetreiver, - maxUpdateLogSize, initUpdateRetryDelay); + maxUpdateLogSize, initUpdateRetryDelay, shouldInit); } private void spawnInitialUpdater(final Updateable<K> updateable, http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java index 3d756c9..142db3f 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java @@ -20,8 +20,6 @@ package org.apache.sentry.hdfs; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; - public class UpdateablePermissions implements Updateable<PermissionsUpdate>{ private static final String UPDATABLE_TYPE_NAME = "perm_update"; @@ -62,5 +60,4 @@ public class UpdateablePermissions implements Updateable<PermissionsUpdate>{ public String getUpdateableTypeName() { return UPDATABLE_TYPE_NAME; } - } http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java index 315d4b3..ac2202f 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java @@ -23,10 +23,10 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.locks.ReadWriteLock; +import org.apache.thrift.TException; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; import org.apache.sentry.hdfs.Updateable.Update; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.After; @@ -77,6 +77,16 @@ public class TestUpdateForwarder { public void deserialize(byte[] data) throws IOException { state = new String(data); } + + @Override + public String serializeToJSON() throws TException { + return state; + } + + @Override + public void deserializeFromJSON(String update) throws TException { + state = new String(update); + } } static class DummyUpdatable implements Updateable<DummyUpdate> { @@ -153,7 +163,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10, true); Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum()); List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); Assert.assertTrue(allUpdates.size() == 1); @@ -173,7 +183,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -194,7 +204,7 @@ public class TestUpdateForwarder { Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED, false)); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), null, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), null, 5, false); updateForwarder.handleUpdateNotification(new DummyUpdate(-1, true).setState("a")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -221,7 +231,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -271,7 +281,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); @@ -317,7 +327,7 @@ public class TestUpdateForwarder { DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); imageRetreiver.setState("a,b,c"); updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5); + testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); while(!updateForwarder.areAllUpdatesCommited()) { Thread.sleep(100); http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java index 2ff715f..db8c334 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java @@ -20,6 +20,7 @@ package org.apache.sentry.provider.db; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.core.common.exception.SentryUserException; +import org.apache.sentry.hdfs.Updateable; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest; @@ -28,6 +29,9 @@ import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleRevokePrivil import org.apache.sentry.provider.db.service.thrift.TDropPrivilegesRequest; import org.apache.sentry.provider.db.service.thrift.TDropSentryRoleRequest; import org.apache.sentry.provider.db.service.thrift.TRenamePrivilegesRequest; +import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; + +import java.util.Map; public interface SentryPolicyStorePlugin { @@ -43,18 +47,18 @@ public interface SentryPolicyStorePlugin { void initialize(Configuration conf, SentryStore sentryStore) throws SentryPluginException; - void onAlterSentryRoleAddGroups(TAlterSentryRoleAddGroupsRequest tRequest) throws SentryPluginException; - - void onAlterSentryRoleDeleteGroups(TAlterSentryRoleDeleteGroupsRequest tRequest) throws SentryPluginException; + Updateable.Update onAlterSentryRoleAddGroups(TAlterSentryRoleAddGroupsRequest tRequest) throws SentryPluginException; - void onAlterSentryRoleGrantPrivilege(TAlterSentryRoleGrantPrivilegeRequest tRequest) throws SentryPluginException; + Updateable.Update onAlterSentryRoleDeleteGroups(TAlterSentryRoleDeleteGroupsRequest tRequest) throws SentryPluginException; - void onAlterSentryRoleRevokePrivilege(TAlterSentryRoleRevokePrivilegeRequest tRequest) throws SentryPluginException; + void onAlterSentryRoleGrantPrivilege(TAlterSentryRoleGrantPrivilegeRequest tRequest, + Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap) throws SentryPluginException; - void onDropSentryRole(TDropSentryRoleRequest tRequest) throws SentryPluginException; + Updateable.Update onAlterSentryRoleRevokePrivilege(TAlterSentryRoleRevokePrivilegeRequest tRequest) throws SentryPluginException; - void onRenameSentryPrivilege(TRenamePrivilegesRequest request) throws SentryPluginException; + Updateable.Update onDropSentryRole(TDropSentryRoleRequest tRequest) throws SentryPluginException; - void onDropSentryPrivilege(TDropPrivilegesRequest request) throws SentryPluginException; + Updateable.Update onRenameSentryPrivilege(TRenamePrivilegesRequest request) throws SentryPluginException; + Updateable.Update onDropSentryPrivilege(TDropPrivilegesRequest request) throws SentryPluginException; } http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java new file mode 100644 index 0000000..bddf628 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.model; + +import javax.jdo.annotations.PersistenceCapable; +import javax.jdo.annotations.PrimaryKey; + +/** + * Database backed Sentry Changes. Any changes to this object + * require re-running the maven build so DN can re-enhance. + */ + +@PersistenceCapable +public class MSentryPathChange { + + @PrimaryKey + private long changeID; + + /** + * Change in Json format + */ + private String pathChange; + private long pathID; + private long createTimeMs; + + public MSentryPathChange(long changeID, long pathID, String pathChange, long createTime) { + this.changeID = changeID; + this.pathID = pathID; + this.pathChange = pathChange; + this.createTimeMs = createTime; + } + + public void setCreateTime(long createTime) { + this.createTimeMs = createTime; + } + + @Override + public String toString() { + return "MSentryChange [changeID=" + changeID + ", pathID= " + pathID + ", pathChange= " + pathChange + ", createTime=" + createTimeMs + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((pathChange == null) ? 0 : pathChange.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + MSentryPathChange other = (MSentryPathChange) obj; + if (changeID != other.changeID) { + return false; + } + + if (pathID != other.pathID) { + return false; + } + + if (!pathChange.equals(other.pathChange)) { + return false; + } + + if (createTimeMs != other.createTimeMs) { + return false; + } + + return true; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java new file mode 100644 index 0000000..c79b0f9 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.model; + +import javax.jdo.annotations.PersistenceCapable; +import javax.jdo.annotations.PrimaryKey; + + +/** + * Database backed Sentry Changes. Any changes to this object + * require re-running the maven build so DN can re-enhance. + */ +@PersistenceCapable +public class MSentryPermChange { + + @PrimaryKey + private long changeID; + + /** + * Change in Json format + */ + private String permChange; + private long createTimeMs; + + public MSentryPermChange(long changeID, String permChange, long createTimeMs) { + this.changeID = changeID; + this.permChange = permChange; + this.createTimeMs = createTimeMs; + } + + public void setCreateTime(long createTimeMs) { + this.createTimeMs = createTimeMs; + } + + public long getChangeID() { + return changeID; + } + + public String getPermChange() { + return permChange; + } + + @Override + public String toString() { + return "MSentryPermChange [changeID=" + changeID + ", permChange= " + permChange + ", createTimeMs=" + createTimeMs + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((permChange == null) ? 0 : permChange.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + MSentryPermChange other = (MSentryPermChange) obj; + if (changeID != other.changeID) { + return false; + } + + if (createTimeMs != other.createTimeMs) { + return false; + } + + if (!permChange.equals(other.permChange)) { + return false; + } + + return true; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo index fb5470f..d166018 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo @@ -262,6 +262,34 @@ </field> </class> + <class name="MSentryPermChange" table="SENTRY_PERM_CHANGE" identity-type="application" detachable="true"> + <field name="changeID" primary-key="true"> + <column name="CHANGE_ID" jdbc-type="BIGINT" allows-null="false"/> + </field> + <field name ="permChange"> + <column name="PERM_CHANGE" length="4000" jdbc-type="VARCHAR" allows-null="false"/> + </field> + <field name="createTimeMs"> + <column name="CREATE_TIME_MS" jdbc-type="BIGINT"/> + </field> + </class> + + <class name="MSentryPathChange" table="SENTRY_PATH_CHANGE" identity-type="application" detachable="true"> + <field name="changeID" primary-key="true"> + <column name="CHANGE_ID" jdbc-type="BIGINT" allows-null="false"/> + </field> + <!-- pathID corresponds to Hive Notification Event ID --> + <field name="pathID"> + <column name="PATH_ID" jdbc-type="BIGINT" allows-null="false"/> + </field> + <field name ="pathChange"> + <column name="PATH_CHANGE" length="4000" jdbc-type="VARCHAR" allows-null="false"/> + </field> + <field name="createTimeMs"> + <column name="CREATE_TIME_MS" jdbc-type="BIGINT"/> + </field> + </class> + </package> </jdo> http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 8bfa78c..b23895f 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -56,12 +56,9 @@ import org.apache.sentry.core.common.exception.SentryAlreadyExistsException; import org.apache.sentry.core.common.exception.SentryGrantDeniedException; import org.apache.sentry.core.common.exception.SentryInvalidInputException; import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; -import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping; -import org.apache.sentry.provider.db.service.model.MSentryGroup; -import org.apache.sentry.provider.db.service.model.MSentryPrivilege; -import org.apache.sentry.provider.db.service.model.MSentryRole; -import org.apache.sentry.provider.db.service.model.MSentryUser; -import org.apache.sentry.provider.db.service.model.MSentryVersion; +import org.apache.sentry.hdfs.PermissionsUpdate; +import org.apache.sentry.hdfs.Updateable; +import org.apache.sentry.provider.db.service.model.*; import org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor; import org.apache.sentry.provider.db.service.thrift.TSentryActiveRoleSet; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; @@ -103,6 +100,9 @@ public class SentryStore { public static int INDEX_GROUP_ROLES_MAP = 0; public static int INDEX_USER_ROLES_MAP = 1; + // Initial change ID for permission/path change. + private static long INIT_CHANGE_ID = -1; + private static final Set<String> ALL_ACTIONS = Sets.newHashSet(AccessConstants.ALL, AccessConstants.SELECT, AccessConstants.INSERT, AccessConstants.ALTER, AccessConstants.CREATE, AccessConstants.DROP, AccessConstants.INDEX, @@ -384,26 +384,49 @@ public class SentryStore { public CommitContext alterSentryRoleGrantPrivilege(String grantorPrincipal, String roleName, TSentryPrivilege privilege) throws Exception { + Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap = Maps.newHashMap(); + privilegesUpdateMap.put(privilege, null); return alterSentryRoleGrantPrivileges(grantorPrincipal, - roleName, Sets.newHashSet(privilege)); + roleName, Sets.newHashSet(privilege), privilegesUpdateMap); } + /** + * Alter sentry Role to grant Privilege, as well as persist the permission change + * to MSentryPermChange table. If commit fails, then retry. + */ public CommitContext alterSentryRoleGrantPrivileges(final String grantorPrincipal, - final String roleName, final Set<TSentryPrivilege> privileges) + final String roleName, final Set<TSentryPrivilege> privileges, final Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap) throws Exception { - return (CommitContext)tm.executeTransactionWithRetry( + return (CommitContext) tm.executeTransactionWithRetry( new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { String trimmedRoleName = trimAndLower(roleName); + for (TSentryPrivilege privilege : privileges) { + Updateable.Update update = privilegesUpdateMap.get(privilege); +// if (update == null) { +// throw new Exception ("Each privileges change for alterSentryRoleGrantPrivileges should have corresponding perm change!"); +// } + // first do grant check grantOptionCheck(pm, grantorPrincipal, privilege); + + // Then read current last processed delta change ID. + long permChangeID = getLastProcessedPermChangeID(pm); + + // Alter sentry Role and grant Privilege. MSentryPrivilege mPrivilege = alterSentryRoleGrantPrivilegeCore( pm, trimmedRoleName, privilege); + if (mPrivilege != null) { + // update the privilege to be the one actually updated. convertToTSentryPrivilege(mPrivilege, privilege); + + // Persist the perm change + persistUpdate(pm, update, permChangeID); } } + return new CommitContext(SERVER_UUID, incrementGetSequenceId()); } }); @@ -2583,4 +2606,85 @@ public class SentryStore { existRoleNames.add(lowerRoleName); } } + + /** + * Persist the delta change. Atomic increase changeID by 1, and persist the change. + * Return if update is null. + */ + private void persistUpdate(PersistenceManager pm, Updateable.Update update, long changeID) throws Exception{ + if (update == null) { + return; + } + + MSentryPermChange mSentryChange = new MSentryPermChange(changeID + 1, update.serializeToJSON(), System.currentTimeMillis()); + pm.makePersistent(mSentryChange); + } + + /** + * Get the last processed perm change. + */ + private long getLastProcessedPermChangeID(PersistenceManager pm) { + Query query = pm.newQuery(MSentryPermChange.class); + query.setResult("max(this.changeID)"); + MSentryPermChange permChange = (MSentryPermChange) query.execute(); + if (permChange == null) { + return INIT_CHANGE_ID; + } else { + return permChange.getChangeID(); + } + } + + /** + * Get the MSentryPermChange object by ChangeID. + */ + public MSentryPermChange getMSentryPermChangeByID(final long changeID) { + MSentryPermChange result = null; + try { + result = (MSentryPermChange) tm.executeTransaction( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + Query query = pm.newQuery(MSentryPermChange.class); + query.setFilter("this.changeID == t"); + query.declareParameters("long t"); + List<MSentryPermChange> permChanges = (List<MSentryPermChange>)query.execute(changeID); + + if (permChanges.size() > 1) { + throw new Exception("Each change ID should only corresponds to one perm change!"); + } + + return permChanges.get(0); + } + }); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + return result; + } + + /** + * Get the MSentryPathChange object by ChangeID. + */ + public MSentryPathChange getMSentryPathChangeByID(final Long changeID) { + MSentryPathChange result = null; + try { + result = (MSentryPathChange) tm.executeTransaction( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + Query query = pm.newQuery(MSentryPathChange.class); + query.setFilter("this.changeID == t"); + query.declareParameters("long t"); + List<MSentryPathChange> pathChanges = (List<MSentryPathChange>)query.execute(changeID); + + if (pathChanges.size() > 1) { + throw new Exception("Each change ID should only corresponds to one path change!"); + } + + return pathChanges.get(0); + } + }); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + return result; + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java index 19daa75..7cac1d2 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.sentry.core.common.exception.SentryUserException; import org.apache.sentry.core.common.exception.SentrySiteConfigurationException; import org.apache.sentry.core.model.db.AccessConstants; +import org.apache.sentry.hdfs.Updateable; import org.apache.sentry.provider.common.GroupMappingService; import org.apache.sentry.core.common.utils.PolicyFileConstants; import org.apache.sentry.core.common.exception.SentryGroupNotFoundException; @@ -258,8 +259,16 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { if (request.isSetPrivilege()) { request.setPrivileges(Sets.newHashSet(request.getPrivilege())); } + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap = null; + for (SentryPolicyStorePlugin plugin : sentryPlugins) { + plugin.onAlterSentryRoleGrantPrivilege(request, privilegesUpdateMap); + } + CommitContext commitContext = sentryStore.alterSentryRoleGrantPrivileges(request.getRequestorUserName(), - request.getRoleName(), request.getPrivileges()); + request.getRoleName(), request.getPrivileges(), privilegesUpdateMap); response.setStatus(Status.OK()); response.setPrivileges(request.getPrivileges()); // Maintain compatibility for old API: Set privilege field to response @@ -268,9 +277,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { } notificationHandlerInvoker.alter_sentry_role_grant_privilege(commitContext, request, response); - for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onAlterSentryRoleGrantPrivilege(request); - } + } catch (SentryNoSuchObjectException e) { String msg = "Role: " + request.getRoleName() + " doesn't exist"; LOGGER.error(msg, e); http://git-wip-us.apache.org/repos/asf/sentry/blob/c22dbe45/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index 7941f9c..1dbd79d 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -35,6 +35,9 @@ import org.apache.sentry.core.model.db.AccessConstants; import org.apache.sentry.core.common.exception.SentryAlreadyExistsException; import org.apache.sentry.core.common.exception.SentryGrantDeniedException; import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; +import org.apache.sentry.hdfs.PermissionsUpdate; +import org.apache.sentry.hdfs.Updateable; +import org.apache.sentry.provider.db.service.model.MSentryPermChange; import org.apache.sentry.provider.db.service.model.MSentryPrivilege; import org.apache.sentry.provider.db.service.model.MSentryRole; import org.apache.sentry.provider.db.service.thrift.TSentryActiveRoleSet; @@ -485,6 +488,7 @@ public class TestSentryStore extends org.junit.Assert { String[] columns = {"c1","c2","c3","c4"}; sentryStore.createSentryRole(roleName); Set<TSentryPrivilege> tPrivileges = Sets.newHashSet(); + Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap = Maps.newHashMap(); for (String column : columns) { TSentryPrivilege privilege = new TSentryPrivilege(); privilege.setPrivilegeScope("Column"); @@ -495,8 +499,9 @@ public class TestSentryStore extends org.junit.Assert { privilege.setAction(AccessConstants.SELECT); privilege.setCreateTime(System.currentTimeMillis()); tPrivileges.add(privilege); + privilegesUpdateMap.put(privilege, null); } - sentryStore.alterSentryRoleGrantPrivileges(grantor, roleName, tPrivileges); + sentryStore.alterSentryRoleGrantPrivileges(grantor, roleName, tPrivileges, privilegesUpdateMap); MSentryRole role = sentryStore.getMSentryRoleByName(roleName); Set<MSentryPrivilege> privileges = role.getPrivileges(); assertEquals(privileges.toString(), 4, privileges.size()); @@ -2058,6 +2063,39 @@ public class TestSentryStore extends org.junit.Assert { } + @Test + public void testGrantPrivilegesWithPermUpdate() throws Exception { + String roleName = "test-privilege"; + String grantor = "g1"; + String server = "server1"; + String db = "db1"; + String table = "tbl1"; + sentryStore.createSentryRole(roleName); + + Map<TSentryPrivilege, Updateable.Update> privilegesUpdateMap = Maps.newHashMap(); + TSentryPrivilege privilege = new TSentryPrivilege(); + privilege.setPrivilegeScope("Column"); + privilege.setServerName(server); + privilege.setDbName(db); + privilege.setTableName(table); + privilege.setAction(AccessConstants.SELECT); + privilege.setCreateTime(System.currentTimeMillis()); + + PermissionsUpdate update = new PermissionsUpdate(0, false); + update.addPrivilegeUpdate("db1.tbl1").putToAddPrivileges( + roleName, privilege.getAction().toUpperCase()); + + privilegesUpdateMap.put(privilege, update); + sentryStore.alterSentryRoleGrantPrivileges(grantor, roleName, Sets.newHashSet(privilege), privilegesUpdateMap); + MSentryRole role = sentryStore.getMSentryRoleByName(roleName); + Set<MSentryPrivilege> privileges = role.getPrivileges(); + assertEquals(privileges.toString(), 1, privileges.size()); + + // Query the persisted perm change and ensure it equals to the original one + MSentryPermChange permChange = sentryStore.getMSentryPermChangeByID(0); + assertEquals(update.serializeToJSON(),permChange.getPermChange()); + } + protected static void addGroupsToUser(String user, String... groupNames) { policyFile.addGroupsToUser(user, groupNames); }
