SENTRY-432. HDFS ingration. Initial patch
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/0eb6645e Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/0eb6645e Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/0eb6645e Branch: refs/heads/sentry-hdfs-sync Commit: 0eb6645e8848681e1746da4d616cb4160c514861 Parents: 00a5eba Author: Prasad Mujumdar <[email protected]> Authored: Mon Sep 15 14:02:07 2014 -0700 Committer: Prasad Mujumdar <[email protected]> Committed: Mon Sep 15 14:07:13 2014 -0700 ---------------------------------------------------------------------- pom.xml | 26 +- sentry-binding/sentry-binding-hive/pom.xml | 8 + .../SentryHiveAuthorizationTaskFactoryImpl.java | 16 + .../sentry/binding/hive/conf/HiveAuthzConf.java | 6 +- .../SentryMetastorePostEventListener.java | 105 +- sentry-dist/pom.xml | 12 + sentry-hdfs-int/pom.xml | 70 + .../hdfs/SentryAuthorizationConstants.java | 55 + .../sentry/hdfs/SentryAuthorizationInfo.java | 233 + .../hdfs/SentryAuthorizationProvider.java | 370 + .../apache/sentry/hdfs/SentryPermissions.java | 159 + .../apache/sentry/hdfs/SentryServiceClient.java | 200 + .../org/apache/sentry/hdfs/SentryUpdater.java | 60 + .../sentry/hdfs/UpdateableAuthzPermissions.java | 179 + .../hdfs/MockSentryAuthorizationProvider.java | 26 + .../sentry/hdfs/SentryAuthorizationInfoX.java | 85 + .../hdfs/TestSentryAuthorizationProvider.java | 163 + .../src/test/resources/hdfs-sentry.xml | 33 + sentry-hdfs/pom.xml | 66 + .../java/org/apache/sentry/hdfs/AuthzPaths.java | 30 + .../apache/sentry/hdfs/AuthzPathsDumper.java | 28 + .../apache/sentry/hdfs/AuthzPermissions.java | 28 + .../sentry/hdfs/ExtendedMetastoreClient.java | 104 + .../java/org/apache/sentry/hdfs/HMSPaths.java | 467 + .../org/apache/sentry/hdfs/HMSPathsSerDe.java | 113 + .../org/apache/sentry/hdfs/MetastoreClient.java | 34 + .../org/apache/sentry/hdfs/PathsUpdate.java | 84 + .../apache/sentry/hdfs/PermissionsUpdate.java | 93 + .../java/org/apache/sentry/hdfs/Updateable.java | 61 + .../sentry/hdfs/UpdateableAuthzPaths.java | 130 + .../org/apache/sentry/hdfs/DummyAdapter.java | 39 + .../apache/sentry/hdfs/DummyAuthzSource.java | 60 + .../org/apache/sentry/hdfs/DummyHMSClient.java | 79 + .../sentry/hdfs/TestAuthzPathCacheOld.java | 523 + .../apache/sentry/hdfs/TestAuthzPermCache.java | 64 + .../org/apache/sentry/hdfs/TestHMSPaths.java | 357 + .../sentry/hdfs/TestHMSPathsFullDump.java | 97 + .../sentry/hdfs/TestUpdateableAuthzPaths.java | 136 + sentry-hdfs/src/test/resources/hdfs-sentry.xml | 22 + sentry-provider/sentry-provider-db/pom.xml | 91 +- .../db/service/thrift/SentryPolicyService.java | 8966 ------------- .../TAlterSentryRoleAddGroupsRequest.java | 742 -- .../TAlterSentryRoleAddGroupsResponse.java | 390 - .../TAlterSentryRoleDeleteGroupsRequest.java | 742 -- .../TAlterSentryRoleDeleteGroupsResponse.java | 390 - .../TAlterSentryRoleGrantPrivilegeRequest.java | 693 - .../TAlterSentryRoleGrantPrivilegeResponse.java | 390 - .../TAlterSentryRoleRevokePrivilegeRequest.java | 693 - ...TAlterSentryRoleRevokePrivilegeResponse.java | 390 - .../thrift/TCreateSentryRoleRequest.java | 587 - .../thrift/TCreateSentryRoleResponse.java | 390 - .../service/thrift/TDropPrivilegesRequest.java | 592 - .../service/thrift/TDropPrivilegesResponse.java | 390 - .../service/thrift/TDropSentryRoleRequest.java | 587 - .../service/thrift/TDropSentryRoleResponse.java | 390 - ...TListSentryPrivilegesForProviderRequest.java | 759 -- ...ListSentryPrivilegesForProviderResponse.java | 543 - .../thrift/TListSentryPrivilegesRequest.java | 702 - .../thrift/TListSentryPrivilegesResponse.java | 554 - .../service/thrift/TListSentryRolesRequest.java | 596 - .../thrift/TListSentryRolesResponse.java | 554 - .../thrift/TRenamePrivilegesRequest.java | 698 - .../thrift/TRenamePrivilegesResponse.java | 390 - .../db/service/thrift/TSentryActiveRoleSet.java | 536 - .../db/service/thrift/TSentryAuthorizable.java | 707 - .../db/service/thrift/TSentryGrantOption.java | 48 - .../db/service/thrift/TSentryGroup.java | 385 - .../db/service/thrift/TSentryPrivilege.java | 1251 -- .../provider/db/service/thrift/TSentryRole.java | 641 - .../service/thrift/TSentryResponseStatus.java | 594 - .../thrift/sentry_common_serviceConstants.java | 50 - .../provider/db/service/UpdateForwarder.java | 227 + .../db/service/UpdateablePermissions.java | 62 + .../db/service/persistent/SentryStore.java | 71 +- .../thrift/SentryPolicyServiceClient.java | 14 +- .../thrift/SentryPolicyStoreProcessor.java | 190 + .../sentry/service/thrift/SentryService.java | 3 +- .../sentry/service/thrift/ServiceConstants.java | 150 - .../main/resources/sentry_common_service.thrift | 42 - .../main/resources/sentry_policy_service.thrift | 223 - .../db/service/TestUpdateForwarder.java | 277 + .../thrift/TestSentryPolicyStoreProcessor.java | 2 + .../thrift/TestSentryServerWithoutKerberos.java | 60 +- sentry-service-client/pom.xml | 164 + .../db/service/thrift/SentryPolicyService.java | 11554 +++++++++++++++++ .../TAlterSentryRoleAddGroupsRequest.java | 742 ++ .../TAlterSentryRoleAddGroupsResponse.java | 390 + .../TAlterSentryRoleDeleteGroupsRequest.java | 742 ++ .../TAlterSentryRoleDeleteGroupsResponse.java | 390 + .../TAlterSentryRoleGrantPrivilegeRequest.java | 693 + .../TAlterSentryRoleGrantPrivilegeResponse.java | 390 + .../TAlterSentryRoleRevokePrivilegeRequest.java | 693 + ...TAlterSentryRoleRevokePrivilegeResponse.java | 390 + .../db/service/thrift/TAuthzUpdateResponse.java | 603 + .../thrift/TCreateSentryRoleRequest.java | 587 + .../thrift/TCreateSentryRoleResponse.java | 390 + .../service/thrift/TDropPrivilegesRequest.java | 592 + .../service/thrift/TDropPrivilegesResponse.java | 390 + .../service/thrift/TDropSentryRoleRequest.java | 587 + .../service/thrift/TDropSentryRoleResponse.java | 390 + ...TListSentryPrivilegesForProviderRequest.java | 759 ++ ...ListSentryPrivilegesForProviderResponse.java | 543 + .../thrift/TListSentryPrivilegesRequest.java | 702 + .../thrift/TListSentryPrivilegesResponse.java | 554 + .../service/thrift/TListSentryRolesRequest.java | 596 + .../thrift/TListSentryRolesResponse.java | 554 + .../db/service/thrift/TPathChanges.java | 765 ++ .../provider/db/service/thrift/TPathEntry.java | 747 ++ .../provider/db/service/thrift/TPathsDump.java | 549 + .../db/service/thrift/TPathsUpdate.java | 748 ++ .../db/service/thrift/TPermissionsUpdate.java | 810 ++ .../db/service/thrift/TPrivilegeChanges.java | 713 + .../thrift/TRenamePrivilegesRequest.java | 698 + .../thrift/TRenamePrivilegesResponse.java | 390 + .../db/service/thrift/TRoleChanges.java | 691 + .../db/service/thrift/TSentryActiveRoleSet.java | 536 + .../db/service/thrift/TSentryAuthorizable.java | 707 + .../db/service/thrift/TSentryGrantOption.java | 48 + .../db/service/thrift/TSentryGroup.java | 385 + .../db/service/thrift/TSentryPrivilege.java | 1251 ++ .../provider/db/service/thrift/TSentryRole.java | 641 + .../service/thrift/TSentryResponseStatus.java | 594 + .../thrift/sentry_common_serviceConstants.java | 50 + .../sentry/service/thrift/ServiceConstants.java | 153 + .../main/resources/sentry_common_service.thrift | 42 + .../main/resources/sentry_policy_service.thrift | 266 + sentry-tests/sentry-tests-hive/pom.xml | 5 + ...actMetastoreTestWithStaticConfiguration.java | 3 + .../e2e/metastore/TestMetastoreEndToEnd.java | 49 +- 129 files changed, 39566 insertions(+), 25843 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e172e92..e66e790 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ limitations under the License. <derby.version>10.10.2.0</derby.version> <commons-cli.version>1.2</commons-cli.version> <hive.version>0.13.1-cdh5.2.0-SNAPSHOT</hive.version> - <hadoop.version>2.3.0-cdh5.1.0-SNAPSHOT</hadoop.version> + <hadoop.version>2.5.0</hadoop.version> <fest.reflect.version>1.4.1</fest.reflect.version> <guava.version>11.0.2</guava.version> <junit.version>4.9</junit.version> @@ -149,6 +149,12 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.version}</version> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> </dependency> @@ -324,6 +330,11 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.sentry</groupId> + <artifactId>sentry-service-client</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> <artifactId>sentry-provider-common</artifactId> <version>${project.version}</version> </dependency> @@ -334,6 +345,16 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs-int</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> <artifactId>sentry-provider-cache</artifactId> <version>${project.version}</version> </dependency> @@ -402,6 +423,9 @@ limitations under the License. <module>sentry-policy</module> <module>sentry-tests</module> <module>sentry-dist</module> + <module>sentry-service-client</module> + <module>sentry-hdfs</module> + <module>sentry-hdfs-int</module> </modules> <build> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-binding/sentry-binding-hive/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/pom.xml b/sentry-binding/sentry-binding-hive/pom.xml index e72b370..aa3a8c9 100644 --- a/sentry-binding/sentry-binding-hive/pom.xml +++ b/sentry-binding/sentry-binding-hive/pom.xml @@ -75,6 +75,14 @@ limitations under the License. <!-- required for SentryGrantRevokeTask --> <dependency> <groupId>org.apache.sentry</groupId> + <artifactId>sentry-service-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> <artifactId>sentry-provider-db</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/SentryHiveAuthorizationTaskFactoryImpl.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/SentryHiveAuthorizationTaskFactoryImpl.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/SentryHiveAuthorizationTaskFactoryImpl.java index f38ee91..dfcf63a 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/SentryHiveAuthorizationTaskFactoryImpl.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/SentryHiveAuthorizationTaskFactoryImpl.java @@ -50,11 +50,14 @@ import org.apache.hadoop.hive.ql.security.authorization.Privilege; import org.apache.hadoop.hive.ql.security.authorization.PrivilegeRegistry; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.sentry.core.model.db.AccessConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; public class SentryHiveAuthorizationTaskFactoryImpl implements HiveAuthorizationTaskFactory { + private static final Logger LOG = LoggerFactory.getLogger(SentryHiveAuthorizationTaskFactoryImpl.class); public SentryHiveAuthorizationTaskFactoryImpl(HiveConf conf, Hive db) { @@ -236,8 +239,11 @@ public class SentryHiveAuthorizationTaskFactoryImpl implements HiveAuthorization private Task<? extends Serializable> analyzeGrantRevokeRole(boolean isGrant, ASTNode ast, HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException { + LOG.debug("## FULL AST : [" + ast.dump() + "]"); + LOG.debug("## CHILD AST : [" + ((ASTNode)ast.getChild(0)).dump() + "]"); List<PrincipalDesc> principalDesc = analyzePrincipalListDef( (ASTNode) ast.getChild(0)); + List<String> roles = new ArrayList<String>(); for (int i = 1; i < ast.getChildCount(); i++) { roles.add(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(i).getText())); @@ -314,18 +320,28 @@ public class SentryHiveAuthorizationTaskFactoryImpl implements HiveAuthorization ASTNode child = (ASTNode) node.getChild(i); PrincipalType type = null; switch (child.getType()) { + case 880: + type = PrincipalType.USER; + break; case HiveParser.TOK_USER: type = PrincipalType.USER; break; + case 685: + type = PrincipalType.GROUP; + break; case HiveParser.TOK_GROUP: type = PrincipalType.GROUP; break; + case 782: + type = PrincipalType.ROLE; + break; case HiveParser.TOK_ROLE: type = PrincipalType.ROLE; break; } String principalName = BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText()); PrincipalDesc principalDesc = new PrincipalDesc(principalName, type); + LOG.debug("## Principal : [ " + principalName + ", " + type + "]"); principalList.add(principalDesc); } return principalList; http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java index 3a83895..65d3933 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java @@ -18,7 +18,7 @@ package org.apache.sentry.binding.hive.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.mortbay.log.Log; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -201,8 +201,8 @@ public class HiveAuthzConf extends Configuration { if (retVal == null) { retVal = AuthzConfVars.getDefault(varName); } else { - Log.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() + - " instead of " + varName); +// Log.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() + +// " instead of " + varName); } } if (retVal == null) { http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java index 0500483..30b68ab 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java @@ -19,34 +19,48 @@ package org.apache.sentry.binding.metastore; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.security.UserGroupInformation; import org.apache.sentry.SentryUserException; -import org.apache.sentry.binding.hive.HiveAuthzBindingHook; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars; import org.apache.sentry.core.common.Authorizable; import org.apache.sentry.core.model.db.Database; import org.apache.sentry.core.model.db.Server; import org.apache.sentry.core.model.db.Table; +import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; +import org.apache.sentry.provider.db.service.thrift.TPathChanges; import org.apache.sentry.service.thrift.SentryServiceClientFactory; +import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; + +import com.google.common.collect.Lists; public class SentryMetastorePostEventListener extends MetaStoreEventListener { private final SentryServiceClientFactory sentryClientFactory; private final HiveAuthzConf authzConf; private final Server server; + // Initialized to some value > 1 so that the first update notification + // will trigger a full Image fetch + private final AtomicInteger seqNum = new AtomicInteger(5); + public SentryMetastorePostEventListener(Configuration config) { super(config); sentryClientFactory = new SentryServiceClientFactory(); @@ -57,6 +71,14 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { @Override public void onCreateTable (CreateTableEvent tableEvent) throws MetaException { + PathsUpdate update = createHMSUpdate(); + if (tableEvent.getTable().getSd().getLocation() != null) { + update.newPathChange( + tableEvent.getTable().getDbName() + "." + + tableEvent.getTable().getTableName()).addToAddPaths( + PathsUpdate.cleanPath(tableEvent.getTable().getSd().getLocation())); + notifySentry(update); + } // drop the privileges on the given table, in case if anything was left // behind during the drop if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) { @@ -68,6 +90,12 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { @Override public void onDropTable(DropTableEvent tableEvent) throws MetaException { + PathsUpdate update = createHMSUpdate(); + update.newPathChange( + tableEvent.getTable().getDbName() + "." + + tableEvent.getTable().getTableName()).addToDelPaths( + Lists.newArrayList(PathsUpdate.ALL_PATHS)); + notifySentry(update); // drop the privileges on the given table if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) { return; @@ -79,6 +107,12 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { @Override public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException { + if (dbEvent.getDatabase().getLocationUri() != null) { + PathsUpdate update = createHMSUpdate(); + update.newPathChange(dbEvent.getDatabase().getName()).addToAddPaths( + PathsUpdate.cleanPath(dbEvent.getDatabase().getLocationUri())); + notifySentry(update); + } // drop the privileges on the database, incase anything left behind during // last drop db if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) { @@ -94,10 +128,14 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { */ @Override public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { + PathsUpdate update = createHMSUpdate(); + update.newPathChange(dbEvent.getDatabase().getName()).addToDelPaths( + Lists.newArrayList(PathsUpdate.ALL_PATHS)); + notifySentry(update); + dropSentryDbPrivileges(dbEvent.getDatabase().getName()); if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) { return; } - dropSentryDbPrivileges(dbEvent.getDatabase().getName()); } /** @@ -106,6 +144,7 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { @Override public void onAlterTable (AlterTableEvent tableEvent) throws MetaException { String oldTableName = null, newTableName = null; + // TODO : notify SentryHMSPathCache if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_ALTER_WITH_POLICY_STORE)) { return; } @@ -121,6 +160,51 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { } } + + + @Override + public void onAddPartition(AddPartitionEvent partitionEvent) + throws MetaException { + PathsUpdate update = createHMSUpdate(); +// TPathChanges pathUpdate = update.newPathChange( +// partitionEvent.getTable().getDbName() + "." +// + partitionEvent.getTable().getTableName()); + Map<String, TPathChanges> pcMap = new HashMap<String, TPathChanges>(); + boolean anyPaths = false; + for (Partition part : partitionEvent.getPartitions()) { + if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { + String authzObj = part.getDbName() + "." + part.getTableName(); + TPathChanges pathUpdate = pcMap.get(authzObj); + if (pathUpdate == null) { + pathUpdate = update.newPathChange(authzObj); + pcMap.put(authzObj, pathUpdate); + } + pathUpdate.addToAddPaths(PathsUpdate + .cleanPath(part.getSd().getLocation())); + anyPaths = true; + } + } + if (anyPaths) { + notifySentry(update); + } + // TODO Auto-generated method stub + super.onAddPartition(partitionEvent); + } + + @Override + public void onDropPartition(DropPartitionEvent partitionEvent) + throws MetaException { + PathsUpdate update = createHMSUpdate(); + update.newPathChange( + partitionEvent.getTable().getDbName() + "." + + partitionEvent.getTable().getTableName()).addToDelPaths( + PathsUpdate.cleanPath(partitionEvent.getPartition().getSd() + .getLocation())); + notifySentry(update); + // TODO Auto-generated method stub + super.onDropPartition(partitionEvent); + } + private SentryPolicyServiceClient getSentryServiceClient() throws MetaException { try { @@ -201,8 +285,25 @@ public class SentryMetastorePostEventListener extends MetaStoreEventListener { } } + private void notifySentry(PathsUpdate update) throws MetaException { + if (!authzConf.getBoolean(ServerConfig.SENTRY_HDFS_INTEGRATION_ENABLE, true)) { + return; + } + try { + getSentryServiceClient().notifyHMSUpdate(update); + } catch (SentryUserException e) { + throw new MetaException("Error sending update to Sentry [" + e.getMessage() + "]"); + } + } + private boolean syncWithPolicyStore(AuthzConfVars syncConfVar) { return "true" .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true"))); } + + private PathsUpdate createHMSUpdate() { + PathsUpdate update = new PathsUpdate(seqNum.incrementAndGet(), false); + return update; + } + } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-dist/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-dist/pom.xml b/sentry-dist/pom.xml index cd7126b..510fd97 100644 --- a/sentry-dist/pom.xml +++ b/sentry-dist/pom.xml @@ -64,6 +64,18 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs-int</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-service-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> <artifactId>sentry-policy-common</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/pom.xml b/sentry-hdfs-int/pom.xml new file mode 100644 index 0000000..7f86186 --- /dev/null +++ b/sentry-hdfs-int/pom.xml @@ -0,0 +1,70 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry</artifactId> + <version>1.5.0-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <artifactId>sentry-hdfs-int</artifactId> + <name>Sentry HDFS Integration Plugin</name> + + <dependencies> + + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs</artifactId> + <version>1.5.0-incubating-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-service-client</artifactId> + <version>1.5.0-incubating-SNAPSHOT</version> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationConstants.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationConstants.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationConstants.java new file mode 100644 index 0000000..9f219ce --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationConstants.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +public class SentryAuthorizationConstants { + + public static final String CONFIG_FILE = "hdfs-sentry.xml"; + + public static final String CONFIG_PREFIX = "sentry.authorization-provider."; + + public static final String HDFS_USER_KEY = CONFIG_PREFIX + "hdfs-user"; + public static final String HDFS_USER_DEFAULT = "hive"; + + public static final String HDFS_GROUP_KEY = CONFIG_PREFIX + "hdfs-group"; + public static final String HDFS_GROUP_DEFAULT = "hive"; + + public static final String HDFS_PERMISSION_KEY = CONFIG_PREFIX + + "hdfs-permission"; + public static final long HDFS_PERMISSION_DEFAULT = 0770; + + public static final String HDFS_PATH_PREFIXES_KEY = CONFIG_PREFIX + + "hdfs-path-prefixes"; + public static final String[] HDFS_PATH_PREFIXES_DEFAULT = new String[0]; + + public static final String CACHE_REFRESH_INTERVAL_KEY = CONFIG_PREFIX + + "cache-refresh-interval.ms"; + public static final int CACHE_REFRESH_INTERVAL_DEFAULT = 500; + + public static final String CACHE_STALE_THRESHOLD_KEY = CONFIG_PREFIX + + "cache-stale-threshold.ms"; + public static final int CACHE_STALE_THRESHOLD_DEFAULT = 60 * 1000; + + public static final String CACHE_REFRESH_RETRY_WAIT_KEY = CONFIG_PREFIX + + "cache-refresh-retry-wait.ms"; + public static final int CACHE_REFRESH_RETRY_WAIT_DEFAULT = 30 * 1000; + + public static final String INCLUDE_HDFS_AUTHZ_AS_ACL_KEY = CONFIG_PREFIX + + "include-hdfs-authz-as-acl"; + public static final boolean INCLUDE_HDFS_AUTHZ_AS_ACL_DEFAULT = true; +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java new file mode 100644 index 0000000..59f4f5e --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.util.StringUtils; +import org.apache.sentry.hdfs.SentryServiceClient.SentryAuthzUpdate; +import org.apache.sentry.hdfs.Updateable.Update; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +public class SentryAuthorizationInfo implements Runnable { + private static Logger LOG = + LoggerFactory.getLogger(SentryAuthorizationInfo.class); + + private SentryUpdater updater; + private volatile UpdateableAuthzPaths authzPaths; + private volatile UpdateableAuthzPermissions authzPermissions; + + private int refreshIntervalMillisec; + private int staleThresholdMillisec; + private int retryWaitMillisec; + private ScheduledExecutorService executor; + private volatile long lastUpdate; + private volatile long waitUntil; + private volatile long lastStaleReport; + // We don't need a re-entrant lock.. but we do need a ReadWriteLock + // Unfortunately, the ReentrantReadWriteLick is the only available + // concrete implementation of a ReadWriteLock. + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + @VisibleForTesting + SentryAuthorizationInfo() {} + + public SentryAuthorizationInfo(Configuration conf) throws Exception { + String[] pathPrefixes = conf.getTrimmedStrings( + SentryAuthorizationConstants.HDFS_PATH_PREFIXES_KEY, + SentryAuthorizationConstants.HDFS_PATH_PREFIXES_DEFAULT); + if (pathPrefixes.length == 0) { + LOG.warn("There are not HDFS path prefixes configured in [{}], " + + "Sentry authorization won't be enforced on any HDFS location", + SentryAuthorizationConstants.HDFS_PATH_PREFIXES_KEY); + } else { + refreshIntervalMillisec = conf.getInt( + SentryAuthorizationConstants.CACHE_REFRESH_INTERVAL_KEY, + SentryAuthorizationConstants.CACHE_REFRESH_INTERVAL_DEFAULT); + staleThresholdMillisec = conf.getInt( + SentryAuthorizationConstants.CACHE_STALE_THRESHOLD_KEY, + SentryAuthorizationConstants.CACHE_STALE_THRESHOLD_DEFAULT); + retryWaitMillisec = conf.getInt( + SentryAuthorizationConstants.CACHE_REFRESH_RETRY_WAIT_KEY, + SentryAuthorizationConstants.CACHE_REFRESH_RETRY_WAIT_DEFAULT); + + LOG.debug("Sentry authorization will enforced in the following HDFS " + + "locations: [{}]", StringUtils.arrayToString(pathPrefixes)); + LOG.debug("Refresh interval [{}]ms, retry wait [{}], stale threshold " + + "[{}]ms", new Object[] + {refreshIntervalMillisec, retryWaitMillisec, staleThresholdMillisec}); + + authzPaths = new UpdateableAuthzPaths(pathPrefixes); + authzPermissions = new UpdateableAuthzPermissions(); + waitUntil = System.currentTimeMillis(); + lastStaleReport = 0; + updater = new SentryUpdater(conf, this); + } + } + + UpdateableAuthzPaths getAuthzPaths() { + return authzPaths; + } + + UpdateableAuthzPermissions getAuthzPermissions() { + return authzPermissions; + } + + private void update() { + SentryAuthzUpdate updates = updater.getUpdates(); + UpdateableAuthzPaths newAuthzPaths = processUpdates( + updates.getPathUpdates(), authzPaths); + UpdateableAuthzPermissions newAuthzPerms = processUpdates( + updates.getPermUpdates(), authzPermissions); + // If there were any FULL updates the returned instance would be + // different + if ((newAuthzPaths != authzPaths)||(newAuthzPerms != authzPermissions)) { + lock.writeLock().lock(); + try { + authzPaths = newAuthzPaths; + LOG.warn("##### FULL Updated paths seq Num [" + authzPaths.getLastUpdatedSeqNum() + "]"); + authzPermissions = newAuthzPerms; + LOG.warn("##### FULL Updated perms seq Num [" + authzPermissions.getLastUpdatedSeqNum() + "]"); + } finally { + lock.writeLock().unlock(); + } + } + + } + + private <K extends Update, V extends Updateable<K>> V processUpdates(List<K> updates, + V updateable) { + // In a list of Updates, if there is a full Update, it will be the first + // one in the List.. all the remaining will be partial updates + if (updates.size() > 0) { + if (updates.get(0).hasFullImage()) { + updateable = (V)updateable.updateFull(updates.remove(0)); + } + // Any more elements ? + if (!updates.isEmpty()) { + updateable.updatePartial(updates, lock); + } + } + return updateable; + } + + public void run() { + try { + // In case of previous preUpdate failure, we sleep for a retry wait + // interval we can do this because we are using a singledthreadedexecutor + // and scheduling the runs with fixed delay. + long currTime = System.currentTimeMillis(); + if (waitUntil > currTime) { + Thread.sleep(waitUntil - currTime); + } + update(); + // we reset lastUpdate only on successful pulling + lastUpdate = System.currentTimeMillis(); + waitUntil = lastUpdate; + } catch (Exception ex) { + LOG.warn("Failed to update, will retry in [{}]ms, error: ", + new Object[]{ retryWaitMillisec, ex.getMessage(), ex}); + waitUntil = System.currentTimeMillis() + retryWaitMillisec; + } + } + + public void start() { + if (authzPaths != null) { + try { + update(); + } catch (Exception ex) { + LOG.warn("Failed to do initial update, will retry in [{}]ms, error: ", + new Object[]{retryWaitMillisec, ex.getMessage(), ex}); + waitUntil = System.currentTimeMillis() + retryWaitMillisec; + } + executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r, SentryAuthorizationInfo.class.getName() + + "-refresher"); + t.setDaemon(true); + return t; + } + } + ); + executor.scheduleWithFixedDelay(this, refreshIntervalMillisec, + refreshIntervalMillisec, TimeUnit.MILLISECONDS); + } + } + + public void stop() { + if (authzPaths != null) { + executor.shutdownNow(); + } + } + + public boolean isStale() { + long now = System.currentTimeMillis(); + boolean stale = now - lastUpdate > staleThresholdMillisec; + if (stale && now - lastStaleReport > 30 * 1000) { + LOG.warn("Authorization information has been stale for [{}]s", + (now - lastUpdate) / 1000); + lastStaleReport = now; + } + return stale; + } + + public boolean isManaged(String[] pathElements) { + lock.readLock().lock(); + try { + return authzPaths.isUnderPrefix(pathElements); + } finally { + lock.readLock().unlock(); + } + } + + public boolean doesBelongToAuthzObject(String[] pathElements) { + lock.readLock().lock(); + try { + return authzPaths.findAuthzObject(pathElements) != null; + } finally { + lock.readLock().unlock(); + } + } + + @SuppressWarnings("unchecked") + public List<AclEntry> getAclEntries(String[] pathElements) { + lock.readLock().lock(); + try { + String authzObj = authzPaths.findAuthzObject(pathElements); + return (authzObj != null) ? authzPermissions.getAcls(authzObj) + : Collections.EMPTY_LIST; + } finally { + lock.readLock().unlock(); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java new file mode 100644 index 0000000..2375d1b --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java @@ -0,0 +1,370 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permission and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.namenode.AclFeature; +import org.apache.hadoop.hdfs.server.namenode.AuthorizationProvider; +import org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider; +import org.apache.hadoop.security.AccessControlException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +public class SentryAuthorizationProvider + implements AuthorizationProvider, Configurable { + + static class SentryAclFeature extends AclFeature { + public SentryAclFeature(ImmutableList<AclEntry> entries) { + super(entries); + } + } + + private static Logger LOG = + LoggerFactory.getLogger(SentryAuthorizationProvider.class); + + private boolean started; + private Configuration conf; + private AuthorizationProvider defaultAuthzProvider; + private String user; + private String group; + private FsPermission permission; + private boolean originalAuthzAsAcl; + private SentryAuthorizationInfo authzInfo; + + public SentryAuthorizationProvider() { + this(null); + } + + @VisibleForTesting + SentryAuthorizationProvider(SentryAuthorizationInfo authzInfo) { + this.authzInfo = authzInfo; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public synchronized void start() { + if (started) { + throw new IllegalStateException("Provider already started"); + } + started = true; + try { + if (!conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, false)) { + throw new RuntimeException("HDFS ACLs must be enabled"); + } + + defaultAuthzProvider = new DefaultAuthorizationProvider(); + defaultAuthzProvider.start(); + // Configuration is read from hdfs-sentry.xml and NN configuration, in + // that order of precedence. + Configuration conf = new Configuration(this.conf); + conf.addResource(SentryAuthorizationConstants.CONFIG_FILE); + user = conf.get(SentryAuthorizationConstants.HDFS_USER_KEY, + SentryAuthorizationConstants.HDFS_USER_DEFAULT); + group = conf.get(SentryAuthorizationConstants.HDFS_GROUP_KEY, + SentryAuthorizationConstants.HDFS_GROUP_DEFAULT); + permission = FsPermission.createImmutable( + (short) conf.getLong(SentryAuthorizationConstants.HDFS_PERMISSION_KEY, + SentryAuthorizationConstants.HDFS_PERMISSION_DEFAULT) + ); + originalAuthzAsAcl = conf.getBoolean( + SentryAuthorizationConstants.INCLUDE_HDFS_AUTHZ_AS_ACL_KEY, + SentryAuthorizationConstants.INCLUDE_HDFS_AUTHZ_AS_ACL_DEFAULT); + + LOG.info("Starting"); + LOG.info("Config: hdfs-user[{}] hdfs-group[{}] hdfs-permission[{}] " + + "include-hdfs-authz-as-acl[{}]", new Object[] + {user, group, permission, originalAuthzAsAcl}); + + if (authzInfo == null) { + authzInfo = new SentryAuthorizationInfo(conf); + } + authzInfo.start(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + @Override + public synchronized void stop() { + LOG.debug("Stopping"); + authzInfo.stop(); + defaultAuthzProvider.stop(); + defaultAuthzProvider = null; + } + + @Override + public void setSnaphottableDirs(Map<INodeAuthorizationInfo, Integer> + snapshotableDirs) { + defaultAuthzProvider.setSnaphottableDirs(snapshotableDirs); + } + + @Override + public void addSnapshottable(INodeAuthorizationInfo dir) { + defaultAuthzProvider.addSnapshottable(dir); + } + + @Override + public void removeSnapshottable(INodeAuthorizationInfo dir) { + defaultAuthzProvider.removeSnapshottable(dir); + } + + @Override + public void createSnapshot(INodeAuthorizationInfo dir, int snapshotId) + throws IOException{ + defaultAuthzProvider.createSnapshot(dir, snapshotId); + } + + @Override + public void removeSnapshot(INodeAuthorizationInfo dir, int snapshotId) + throws IOException { + defaultAuthzProvider.removeSnapshot(dir, snapshotId); + } + + @Override + public void checkPermission(String user, Set<String> groups, + INodeAuthorizationInfo[] inodes, int snapshotId, + boolean doCheckOwner, FsAction ancestorAccess, FsAction parentAccess, + FsAction access, FsAction subAccess, boolean ignoreEmptyDir) + throws AccessControlException, UnresolvedLinkException { + defaultAuthzProvider.checkPermission(user, groups, inodes, snapshotId, + doCheckOwner, ancestorAccess, parentAccess, access, subAccess, + ignoreEmptyDir); + } + + private static final String[] EMPTY_STRING_ARRAY = new String[0]; + + private String[] getPathElements(INodeAuthorizationInfo node) { + return getPathElements(node, 0); + } + + private String[] getPathElements(INodeAuthorizationInfo node, int idx) { + String[] paths; + INodeAuthorizationInfo parent = node.getParent(); + if (parent == null) { + paths = (idx > 0) ? new String[idx] : EMPTY_STRING_ARRAY; + } else { + paths = getPathElements(parent, idx + 1); + paths[paths.length - 1 - idx] = node.getLocalName(); + } + return paths; + } + + @Override + public void setUser(INodeAuthorizationInfo node, String user) { + defaultAuthzProvider.setUser(node, user); + } + + @Override + public String getUser(INodeAuthorizationInfo node, int snapshotId) { + String user; + String[] pathElements = getPathElements(node); + if (!authzInfo.isManaged(pathElements)) { + user = defaultAuthzProvider.getUser(node, snapshotId); + } else { + if (!authzInfo.isStale()) { + if (authzInfo.doesBelongToAuthzObject(pathElements)) { + user = this.user; + } else { + user = defaultAuthzProvider.getUser(node, snapshotId); + } + } else { + user = this.user; + } + } + return user; + } + + @Override + public void setGroup(INodeAuthorizationInfo node, String group) { + defaultAuthzProvider.setGroup(node, group); + } + + @Override + public String getGroup(INodeAuthorizationInfo node, int snapshotId) { + String group; + String[] pathElements = getPathElements(node); + if (!authzInfo.isManaged(pathElements)) { + group = defaultAuthzProvider.getGroup(node, snapshotId); + } else { + if (!authzInfo.isStale()) { + if (authzInfo.doesBelongToAuthzObject(pathElements)) { + group = this.group; + } else { + group = defaultAuthzProvider.getGroup(node, snapshotId); + } + } else { + group = this.group; + } + } + return group; + } + + @Override + public void setPermission(INodeAuthorizationInfo node, + FsPermission permission) { + defaultAuthzProvider.setPermission(node, permission); + } + + @Override + public FsPermission getFsPermission( + INodeAuthorizationInfo node, int snapshotId) { + FsPermission permission; + String[] pathElements = getPathElements(node); + if (!authzInfo.isManaged(pathElements)) { + permission = defaultAuthzProvider.getFsPermission(node, snapshotId); + } else { + if (!authzInfo.isStale()) { + if (authzInfo.doesBelongToAuthzObject(pathElements)) { + permission = this.permission; + } else { + permission = defaultAuthzProvider.getFsPermission(node, snapshotId); + } + } else { + permission = this.permission; + } + } + return permission; + } + + private List<AclEntry> createAclEntries(String user, String group, + FsPermission permission) { + List<AclEntry> list = new ArrayList<AclEntry>(); + AclEntry.Builder builder = new AclEntry.Builder(); + FsPermission fsPerm = new FsPermission(permission); + builder.setName(user); + builder.setType(AclEntryType.USER); + builder.setScope(AclEntryScope.ACCESS); + builder.setPermission(fsPerm.getUserAction()); + list.add(builder.build()); + builder.setName(group); + builder.setType(AclEntryType.GROUP); + builder.setScope(AclEntryScope.ACCESS); + builder.setPermission(fsPerm.getGroupAction()); + list.add(builder.build()); + builder.setName(null); + builder.setType(AclEntryType.OTHER); + builder.setScope(AclEntryScope.ACCESS); + builder.setPermission(fsPerm.getOtherAction()); + list.add(builder.build()); + return list; + } + + @Override + public AclFeature getAclFeature(INodeAuthorizationInfo node, int snapshotId) { + AclFeature f = null; + String[] pathElements = getPathElements(node); + String p = Arrays.toString(pathElements); + boolean isManaged = false; + boolean isStale = false; + boolean hasAuthzObj = false; + if (!authzInfo.isManaged(pathElements)) { + isManaged = false; + f = defaultAuthzProvider.getAclFeature(node, snapshotId); + } else { + isManaged = true; + List<AclEntry> list = new ArrayList<AclEntry>(); + if (originalAuthzAsAcl) { + String user = defaultAuthzProvider.getUser(node, snapshotId); + String group = defaultAuthzProvider.getGroup(node, snapshotId); + INodeAuthorizationInfo pNode = node.getParent(); + while (group == null || pNode != null) { + group = defaultAuthzProvider.getGroup(pNode, snapshotId); + pNode = pNode.getParent(); + } + FsPermission perm = defaultAuthzProvider.getFsPermission(node, snapshotId); + list.addAll(createAclEntries(user, group, perm)); + } + if (!authzInfo.isStale()) { + isStale = false; + if (authzInfo.doesBelongToAuthzObject(pathElements)) { + hasAuthzObj = true; + list.addAll(authzInfo.getAclEntries(pathElements)); + f = new SentryAclFeature(ImmutableList.copyOf(list)); + } else { + hasAuthzObj = false; + f = defaultAuthzProvider.getAclFeature(node, snapshotId); + } + } else { + isStale = true; + f = new SentryAclFeature(ImmutableList.copyOf(list)); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("### getAclEntry [" + (p == null ? "null" : p) + "] : [" + + "isManaged=" + isManaged + + ",isStale=" + isStale + + ",hasAuthzObj=" + hasAuthzObj + + ",origAtuhzAsAcl=" + originalAuthzAsAcl + "]" + + "[" + (f == null ? "null" : f.getEntries()) + "]"); + } + return f; + } + + @Override + public void removeAclFeature(INodeAuthorizationInfo node) { + AclFeature aclFeature = node.getAclFeature(CURRENT_STATE_ID); + if (aclFeature.getClass() != SentryAclFeature.class) { + defaultAuthzProvider.removeAclFeature(node); + } + } + + @Override + public void addAclFeature(INodeAuthorizationInfo node, AclFeature f) { + String[] pathElements = getPathElements(node); + if (!authzInfo.isManaged(pathElements)) { + defaultAuthzProvider.addAclFeature(node, f); + } + } + + @Override + public boolean doesAllowChanges(INodeAuthorizationInfo node) { + String[] pathElements = getPathElements(node); + if (!authzInfo.isManaged(pathElements)) { + return defaultAuthzProvider.doesAllowChanges(node); + } + return !authzInfo.doesBelongToAuthzObject(getPathElements(node)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java new file mode 100644 index 0000000..7461f89 --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; + +import com.google.common.collect.Lists; + +public class SentryPermissions implements AuthzPermissions { + + public static class PrivilegeInfo { + private final String authzObj; + private final Map<String, FsAction> roleToPermission = new HashMap<String, FsAction>(); + public PrivilegeInfo(String authzObj) { + this.authzObj = authzObj; + } + public PrivilegeInfo setPermission(String role, FsAction perm) { + roleToPermission.put(role, perm); + return this; + } + public PrivilegeInfo removePermission(String role) { + roleToPermission.remove(role); + return this; + } + public FsAction getPermission(String role) { + return roleToPermission.get(role); + } + public Map<String, FsAction> getAllPermissions() { + return roleToPermission; + } + public String getAuthzObj() { + return authzObj; + } + } + + public static class RoleInfo { + private final String role; + private final Set<String> groups = new HashSet<String>(); + public RoleInfo(String role) { + this.role = role; + } + public RoleInfo addGroup(String group) { + groups.add(group); + return this; + } + public RoleInfo delGroup(String group) { + groups.remove(group); + return this; + } + public String getRole() { + return role; + } + public Set<String> getAllGroups() { + return groups; + } + } + + private final Map<String, PrivilegeInfo> privileges = new HashMap<String, PrivilegeInfo>(); + private final Map<String, RoleInfo> roles = new HashMap<String, RoleInfo>(); + + @Override + public List<AclEntry> getAcls(String authzObj) { + PrivilegeInfo privilegeInfo = privileges.get(authzObj); + Map<String, FsAction> groupPerms = new HashMap<String, FsAction>(); + if (privilegeInfo != null) { + for (Map.Entry<String, FsAction> privs : privilegeInfo + .getAllPermissions().entrySet()) { + constructAclEntry(privs.getKey(), privs.getValue(), groupPerms); + } + } + List<AclEntry> retList = new LinkedList<AclEntry>(); + for (Map.Entry<String, FsAction> groupPerm : groupPerms.entrySet()) { + AclEntry.Builder builder = new AclEntry.Builder(); + builder.setName(groupPerm.getKey()); + builder.setType(AclEntryType.GROUP); + builder.setScope(AclEntryScope.ACCESS); + FsAction action = groupPerm.getValue(); + if ((action == FsAction.READ) || (action == FsAction.WRITE) + || (action == FsAction.READ_WRITE)) { + action = action.or(FsAction.EXECUTE); + } + builder.setPermission(action); + retList.add(builder.build()); + } + return retList; + } + + private void constructAclEntry(String role, FsAction permission, + Map<String, FsAction> groupPerms) { + RoleInfo roleInfo = roles.get(role); + if (roleInfo != null) { + for (String group : roleInfo.groups) { + FsAction fsAction = groupPerms.get(group); + if (fsAction == null) { + fsAction = FsAction.NONE; + } + groupPerms.put(group, fsAction.or(permission)); + } + } + } + + public PrivilegeInfo getPrivilegeInfo(String authzObj) { + return privileges.get(authzObj); + } + + Collection<PrivilegeInfo> getAllPrivileges() { + return privileges.values(); + } + + Collection<RoleInfo> getAllRoles() { + return roles.values(); + } + + public void delPrivilegeInfo(String authzObj) { + privileges.remove(authzObj); + } + + public void addPrivilegeInfo(PrivilegeInfo privilegeInfo) { + privileges.put(privilegeInfo.authzObj, privilegeInfo); + } + + public RoleInfo getRoleInfo(String role) { + return roles.get(role); + } + + public void delRoleInfo(String role) { + roles.remove(role); + } + + public void addRoleInfo(RoleInfo roleInfo) { + roles.put(roleInfo.role, roleInfo); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryServiceClient.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryServiceClient.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryServiceClient.java new file mode 100644 index 0000000..97da9aa --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryServiceClient.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.PrivilegedExceptionAction; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.security.auth.callback.CallbackHandler; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.sentry.provider.db.service.thrift.SentryPolicyService; +import org.apache.sentry.provider.db.service.thrift.SentryPolicyService.Client; +import org.apache.sentry.provider.db.service.thrift.TAuthzUpdateResponse; +import org.apache.sentry.provider.db.service.thrift.TPathsUpdate; +import org.apache.sentry.provider.db.service.thrift.TPermissionsUpdate; +import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig; +import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TMultiplexedProtocol; +import org.apache.thrift.transport.TSaslClientTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public class SentryServiceClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(SentryServiceClient.class); + + public static class SentryAuthzUpdate { + + private final List<PermissionsUpdate> permUpdates; + private final List<PathsUpdate> pathUpdates; + + public SentryAuthzUpdate(List<PermissionsUpdate> permUpdates, List<PathsUpdate> pathUpdates) { + this.permUpdates = permUpdates; + this.pathUpdates = pathUpdates; + } + + public List<PermissionsUpdate> getPermUpdates() { + return permUpdates; + } + + public List<PathsUpdate> getPathUpdates() { + return pathUpdates; + } + } + + /** + * This transport wraps the Sasl transports to set up the right UGI context for open(). + */ + public static class UgiSaslClientTransport extends TSaslClientTransport { + protected UserGroupInformation ugi = null; + + public UgiSaslClientTransport(String mechanism, String authorizationId, + String protocol, String serverName, Map<String, String> props, + CallbackHandler cbh, TTransport transport, boolean wrapUgi) + throws IOException { + super(mechanism, authorizationId, protocol, serverName, props, cbh, + transport); + if (wrapUgi) { + ugi = UserGroupInformation.getLoginUser(); + } + } + + // open the SASL transport with using the current UserGroupInformation + // This is needed to get the current login context stored + @Override + public void open() throws TTransportException { + if (ugi == null) { + baseOpen(); + } else { + try { + ugi.doAs(new PrivilegedExceptionAction<Void>() { + public Void run() throws TTransportException { + baseOpen(); + return null; + } + }); + } catch (IOException e) { + throw new TTransportException("Failed to open SASL transport", e); + } catch (InterruptedException e) { + throw new TTransportException( + "Interrupted while opening underlying transport", e); + } + } + } + + private void baseOpen() throws TTransportException { + super.open(); + } + } + + private final Configuration conf; + private final InetSocketAddress serverAddress; + private final int connectionTimeout; + private boolean kerberos; + private TTransport transport; + + private String[] serverPrincipalParts; + private Client client; + + public SentryServiceClient(Configuration conf) throws IOException { + this.conf = conf; + Preconditions.checkNotNull(this.conf, "Configuration object cannot be null"); + this.serverAddress = NetUtils.createSocketAddr(Preconditions.checkNotNull( + conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key " + + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt( + ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT)); + this.connectionTimeout = conf.getInt(ClientConfig.SERVER_RPC_CONN_TIMEOUT, + ClientConfig.SERVER_RPC_CONN_TIMEOUT_DEFAULT); + kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase( + conf.get(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_KERBEROS).trim()); + transport = new TSocket(serverAddress.getHostName(), + serverAddress.getPort(), connectionTimeout); + if (kerberos) { + String serverPrincipal = Preconditions.checkNotNull(conf.get(ServerConfig.PRINCIPAL), ServerConfig.PRINCIPAL + " is required"); + + // Resolve server host in the same way as we are doing on server side + serverPrincipal = SecurityUtil.getServerPrincipal(serverPrincipal, serverAddress.getAddress()); + LOGGER.info("Using server kerberos principal: " + serverPrincipal); + + serverPrincipalParts = SaslRpcServer.splitKerberosName(serverPrincipal); + Preconditions.checkArgument(serverPrincipalParts.length == 3, + "Kerberos principal should have 3 parts: " + serverPrincipal); + boolean wrapUgi = "true".equalsIgnoreCase(conf + .get(ServerConfig.SECURITY_USE_UGI_TRANSPORT, "true")); + transport = new UgiSaslClientTransport(AuthMethod.KERBEROS.getMechanismName(), + null, serverPrincipalParts[0], serverPrincipalParts[1], + ClientConfig.SASL_PROPERTIES, null, transport, wrapUgi); + } else { + serverPrincipalParts = null; + } + try { + transport.open(); + } catch (TTransportException e) { + throw new IOException("Transport exception while opening transport: " + e.getMessage(), e); + } + LOGGER.info("Successfully opened transport: " + transport + " to " + serverAddress); + TMultiplexedProtocol protocol = new TMultiplexedProtocol( + new TCompactProtocol(transport), + "SentryPolicyService"); + client = new SentryPolicyService.Client(protocol); + LOGGER.info("Successfully created client"); + } + + public synchronized SentryAuthzUpdate getAllUpdatesFrom(long permSeqNum, long pathSeqNum) + throws IOException { + SentryAuthzUpdate retVal = new SentryAuthzUpdate(new LinkedList<PermissionsUpdate>(), new LinkedList<PathsUpdate>()); + try { + TAuthzUpdateResponse sentryUpdates = client.get_all_authz_updates_from(permSeqNum, pathSeqNum); + if (sentryUpdates.getAuthzPathUpdate() != null) { + for (TPathsUpdate pathsUpdate : sentryUpdates.getAuthzPathUpdate()) { + retVal.getPathUpdates().add(new PathsUpdate(pathsUpdate)); + } + } + if (sentryUpdates.getAuthzPermUpdate() != null) { + for (TPermissionsUpdate permsUpdate : sentryUpdates.getAuthzPermUpdate()) { + retVal.getPermUpdates().add(new PermissionsUpdate(permsUpdate)); + } + } + } catch (Exception e) { + throw new IOException("Thrift Exception occurred !!", e); + } + return retVal; + } + + public void close() { + if (transport != null) { + transport.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryUpdater.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryUpdater.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryUpdater.java new file mode 100644 index 0000000..bc46651 --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/SentryUpdater.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sentry.hdfs.SentryServiceClient.SentryAuthzUpdate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SentryUpdater { + + private SentryServiceClient sentryClient; + private final Configuration conf; + private final SentryAuthorizationInfo authzInfo; + + private static Logger LOG = LoggerFactory.getLogger(SentryUpdater.class); + + public SentryUpdater(Configuration conf, SentryAuthorizationInfo authzInfo) throws Exception { + this.conf = conf; + this.authzInfo = authzInfo; + } + + public SentryAuthzUpdate getUpdates() { + if (sentryClient == null) { + try { + sentryClient = new SentryServiceClient(conf); + } catch (Exception e) { + LOG.error("Error connecting to Sentry ['{}'] !!", + e.getMessage()); + return null; + } + } + try { + SentryAuthzUpdate sentryUpdates = sentryClient.getAllUpdatesFrom( + authzInfo.getAuthzPermissions().getLastUpdatedSeqNum() + 1, + authzInfo.getAuthzPaths().getLastUpdatedSeqNum() + 1); + return sentryUpdates; + } catch (Exception e) { + sentryClient = null; + LOG.error("Error receiving updates from Sentry !!", e); + return null; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java new file mode 100644 index 0000000..f9e1cf4 --- /dev/null +++ b/sentry-hdfs-int/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; + +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.sentry.hdfs.SentryPermissions.PrivilegeInfo; +import org.apache.sentry.hdfs.SentryPermissions.RoleInfo; +import org.apache.sentry.provider.db.service.thrift.TPrivilegeChanges; +import org.apache.sentry.provider.db.service.thrift.TRoleChanges; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UpdateableAuthzPermissions implements AuthzPermissions, Updateable<PermissionsUpdate> { + private volatile SentryPermissions perms = new SentryPermissions(); + private final AtomicLong seqNum = new AtomicLong(0); + + private static Logger LOG = LoggerFactory.getLogger(UpdateableAuthzPermissions.class); + + @Override + public List<AclEntry> getAcls(String authzObj) { + return perms.getAcls(authzObj); + } + + @Override + public UpdateableAuthzPermissions updateFull(PermissionsUpdate update) { + UpdateableAuthzPermissions other = new UpdateableAuthzPermissions(); + other.applyPartialUpdate(update); + other.seqNum.set(update.getSeqNum()); + return other; + } + + @Override + public void updatePartial(Iterable<PermissionsUpdate> updates, ReadWriteLock lock) { + lock.writeLock().lock(); + try { + int counter = 0; + for (PermissionsUpdate update : updates) { + applyPartialUpdate(update); + if (++counter > 99) { + counter = 0; + lock.writeLock().unlock(); + lock.writeLock().lock(); + } + seqNum.set(update.getSeqNum()); + LOG.warn("##### Updated perms seq Num [" + seqNum.get() + "]"); + } + } finally { + lock.writeLock().unlock(); + } + } + + + private void applyPartialUpdate(PermissionsUpdate update) { + applyPrivilegeUpdates(update); + applyRoleUpdates(update); + } + + private void applyRoleUpdates(PermissionsUpdate update) { + for (TRoleChanges rUpdate : update.getRoleUpdates()) { + if (rUpdate.getRole().equals(PermissionsUpdate.ALL_ROLES)) { + // Request to remove group from all roles + String groupToRemove = rUpdate.getDelGroups().iterator().next(); + for (RoleInfo rInfo : perms.getAllRoles()) { + rInfo.delGroup(groupToRemove); + } + } + RoleInfo rInfo = perms.getRoleInfo(rUpdate.getRole()); + for (String group : rUpdate.getAddGroups()) { + if (rInfo == null) { + rInfo = new RoleInfo(rUpdate.getRole()); + } + rInfo.addGroup(group); + } + if (rInfo != null) { + perms.addRoleInfo(rInfo); + for (String group : rUpdate.getDelGroups()) { + if (group.equals(PermissionsUpdate.ALL_GROUPS)) { + perms.delRoleInfo(rInfo.getRole()); + break; + } + // If there are no groups to remove, rUpdate.getDelGroups() will + // return empty list and this code will not be reached + rInfo.delGroup(group); + } + } + } + } + + private void applyPrivilegeUpdates(PermissionsUpdate update) { + for (TPrivilegeChanges pUpdate : update.getPrivilegeUpdates()) { + if (pUpdate.getAuthzObj().equals(PermissionsUpdate.ALL_PRIVS)) { + // Request to remove role from all Privileges + String roleToRemove = pUpdate.getDelPrivileges().keySet().iterator() + .next(); + for (PrivilegeInfo pInfo : perms.getAllPrivileges()) { + pInfo.removePermission(roleToRemove); + } + } + PrivilegeInfo pInfo = perms.getPrivilegeInfo(pUpdate.getAuthzObj()); + for (Map.Entry<String, String> aMap : pUpdate.getAddPrivileges().entrySet()) { + if (pInfo == null) { + pInfo = new PrivilegeInfo(pUpdate.getAuthzObj()); + } + FsAction fsAction = pInfo.getPermission(aMap.getKey()); + if (fsAction == null) { + fsAction = FsAction.getFsAction(aMap.getValue()); + } else { + fsAction = fsAction.or(FsAction.getFsAction(aMap.getValue())); + } + pInfo.setPermission(aMap.getKey(), fsAction); + } + if (pInfo != null) { + perms.addPrivilegeInfo(pInfo); + for (Map.Entry<String, String> dMap : pUpdate.getDelPrivileges().entrySet()) { + if (dMap.getKey().equals(PermissionsUpdate.ALL_ROLES)) { + // Remove all privileges + perms.delPrivilegeInfo(pUpdate.getAuthzObj()); + break; + } + FsAction fsAction = pInfo.getPermission(dMap.getKey()); + if (fsAction != null) { + fsAction = fsAction.and(FsAction.getFsAction(dMap.getValue()).not()); + if (FsAction.NONE == fsAction) { + pInfo.removePermission(dMap.getKey()); + } else { + pInfo.setPermission(dMap.getKey(), fsAction); + } + } + } + } + } + } + + @Override + public long getLastUpdatedSeqNum() { + return seqNum.get(); + } + + @Override + public PermissionsUpdate createFullImageUpdate(long currSeqNum) { + PermissionsUpdate retVal = new PermissionsUpdate(currSeqNum, true); + for (PrivilegeInfo pInfo : perms.getAllPrivileges()) { + TPrivilegeChanges pUpdate = retVal.addPrivilegeUpdate(pInfo.getAuthzObj()); + for (Map.Entry<String, FsAction> ent : pInfo.getAllPermissions().entrySet()) { + pUpdate.putToAddPrivileges(ent.getKey(), ent.getValue().SYMBOL); + } + } + for (RoleInfo rInfo : perms.getAllRoles()) { + TRoleChanges rUpdate = retVal.addRoleUpdate(rInfo.getRole()); + for (String group : rInfo.getAllGroups()) { + rUpdate.addToAddGroups(group); + } + } + return retVal; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/MockSentryAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/MockSentryAuthorizationProvider.java b/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/MockSentryAuthorizationProvider.java new file mode 100644 index 0000000..2085b52 --- /dev/null +++ b/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/MockSentryAuthorizationProvider.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +public class MockSentryAuthorizationProvider extends + SentryAuthorizationProvider { + + public MockSentryAuthorizationProvider() { + super(new SentryAuthorizationInfoX()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/SentryAuthorizationInfoX.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/SentryAuthorizationInfoX.java b/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/SentryAuthorizationInfoX.java new file mode 100644 index 0000000..7a1539b --- /dev/null +++ b/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/SentryAuthorizationInfoX.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; + +public class SentryAuthorizationInfoX extends SentryAuthorizationInfo { + + public SentryAuthorizationInfoX() { + super(); + } + + @Override + public void run() { + + } + + @Override + public void start() { + + } + + @Override + public void stop() { + + } + + @Override + public boolean isStale() { + return false; + } + + private static final String[] MANAGED = {"user", "authz"}; + private static final String[] AUTHZ_OBJ = {"user", "authz", "obj"}; + + private boolean hasPrefix(String[] prefix, String[] pathElement) { + int i = 0; + for (; i < prefix.length && i < pathElement.length; i ++) { + if (!prefix[i].equals(pathElement[i])) { + return false; + } + } + return (i == prefix.length); + } + + @Override + public boolean isManaged(String[] pathElements) { + return hasPrefix(MANAGED, pathElements); + } + + @Override + public boolean doesBelongToAuthzObject(String[] pathElements) { + return hasPrefix(AUTHZ_OBJ, pathElements); + } + + @Override + public List<AclEntry> getAclEntries(String[] pathElements) { + AclEntry acl = new AclEntry.Builder().setType(AclEntryType.USER). + setPermission(FsAction.ALL).setName("user-authz"). + setScope(AclEntryScope.ACCESS).build(); + return Arrays.asList(acl); + } +}
