http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/TestSentryAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/TestSentryAuthorizationProvider.java b/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/TestSentryAuthorizationProvider.java new file mode 100644 index 0000000..67919fa --- /dev/null +++ b/sentry-hdfs-int/src/test/java/org/apache/sentry/hdfs/TestSentryAuthorizationProvider.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + +public class TestSentryAuthorizationProvider { + private MiniDFSCluster miniDFS; + private UserGroupInformation admin; + + @Before + public void setUp() throws Exception { + admin = UserGroupInformation.createUserForTesting( + System.getProperty("user.name"), new String[] { "supergroup" }); + admin.doAs(new PrivilegedExceptionAction<Void>() { + @Override + public Void run() throws Exception { + System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, "target/test/data"); + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_AUTHORIZATION_PROVIDER_KEY, + MockSentryAuthorizationProvider.class.getName()); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + miniDFS = new MiniDFSCluster.Builder(conf).build(); + return null; + } + }); + } + + @After + public void cleanUp() throws IOException { + if (miniDFS != null) { + miniDFS.shutdown(); + } + } + + @Test + public void testProvider() throws Exception { + admin.doAs(new PrivilegedExceptionAction<Void>() { + @Override + public Void run() throws Exception { + String sysUser = UserGroupInformation.getCurrentUser().getShortUserName(); + FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0)); + + List<AclEntry> baseAclList = new ArrayList<AclEntry>(); + AclEntry.Builder builder = new AclEntry.Builder(); + baseAclList.add(builder.setType(AclEntryType.USER) + .setScope(AclEntryScope.ACCESS).build()); + baseAclList.add(builder.setType(AclEntryType.GROUP) + .setScope(AclEntryScope.ACCESS).build()); + baseAclList.add(builder.setType(AclEntryType.OTHER) + .setScope(AclEntryScope.ACCESS).build()); + Path path1 = new Path("/user/authz/obj/xxx"); + fs.mkdirs(path1); + fs.setAcl(path1, baseAclList); + + fs.mkdirs(new Path("/user/authz/xxx")); + fs.mkdirs(new Path("/user/xxx")); + + // root + Path path = new Path("/"); + Assert.assertEquals(sysUser, fs.getFileStatus(path).getOwner()); + Assert.assertEquals("supergroup", fs.getFileStatus(path).getGroup()); + Assert.assertEquals(new FsPermission((short) 0755), fs.getFileStatus(path).getPermission()); + Assert.assertTrue(fs.getAclStatus(path).getEntries().isEmpty()); + + // dir before prefixes + path = new Path("/user"); + Assert.assertEquals(sysUser, fs.getFileStatus(path).getOwner()); + Assert.assertEquals("supergroup", fs.getFileStatus(path).getGroup()); + Assert.assertEquals(new FsPermission((short) 0755), fs.getFileStatus(path).getPermission()); + Assert.assertTrue(fs.getAclStatus(path).getEntries().isEmpty()); + + // prefix dir + path = new Path("/user/authz"); + Assert.assertEquals(sysUser, fs.getFileStatus(path).getOwner()); + Assert.assertEquals("supergroup", fs.getFileStatus(path).getGroup()); + Assert.assertEquals(new FsPermission((short) 0755), fs.getFileStatus(path).getPermission()); + Assert.assertTrue(fs.getAclStatus(path).getEntries().isEmpty()); + + // dir inside of prefix, no obj + path = new Path("/user/authz/xxx"); + FileStatus status = fs.getFileStatus(path); + Assert.assertEquals(sysUser, status.getOwner()); + Assert.assertEquals("supergroup", status.getGroup()); + Assert.assertEquals(new FsPermission((short) 0755), status.getPermission()); + Assert.assertTrue(fs.getAclStatus(path).getEntries().isEmpty()); + + // dir inside of prefix, obj + path = new Path("/user/authz/obj"); + Assert.assertEquals("hive", fs.getFileStatus(path).getOwner()); + Assert.assertEquals("hive", fs.getFileStatus(path).getGroup()); + Assert.assertEquals(new FsPermission((short) 0770), fs.getFileStatus(path).getPermission()); + Assert.assertFalse(fs.getAclStatus(path).getEntries().isEmpty()); + + List<AclEntry> acls = new ArrayList<AclEntry>(); + acls.add(new AclEntry.Builder().setName(sysUser).setType(AclEntryType.USER).setScope(AclEntryScope.ACCESS).setPermission(FsAction.ALL).build()); + acls.add(new AclEntry.Builder().setName("supergroup").setType(AclEntryType.GROUP).setScope(AclEntryScope.ACCESS).setPermission(FsAction.READ_EXECUTE).build()); + acls.add(new AclEntry.Builder().setName(null).setType(AclEntryType.OTHER).setScope(AclEntryScope.ACCESS).setPermission(FsAction.READ_EXECUTE).build()); + acls.add(new AclEntry.Builder().setName("user-authz").setType(AclEntryType.USER).setScope(AclEntryScope.ACCESS).setPermission(FsAction.ALL).build()); + Assert.assertEquals(new LinkedHashSet<AclEntry>(acls), new LinkedHashSet<AclEntry>(fs.getAclStatus(path).getEntries())); + + // dir inside of prefix, inside of obj + path = new Path("/user/authz/obj/xxx"); + Assert.assertEquals("hive", fs.getFileStatus(path).getOwner()); + Assert.assertEquals("hive", fs.getFileStatus(path).getGroup()); + Assert.assertEquals(new FsPermission((short) 0770), fs.getFileStatus(path).getPermission()); + Assert.assertFalse(fs.getAclStatus(path).getEntries().isEmpty()); + + Path path2 = new Path("/user/authz/obj/path2"); + fs.mkdirs(path2); + fs.setAcl(path2, baseAclList); + + // dir outside of prefix + path = new Path("/user/xxx"); + Assert.assertEquals(sysUser, fs.getFileStatus(path).getOwner()); + Assert.assertEquals("supergroup", fs.getFileStatus(path).getGroup()); + Assert.assertEquals(new FsPermission((short) 0755), fs.getFileStatus(path).getPermission()); + Assert.assertTrue(fs.getAclStatus(path).getEntries().isEmpty()); + return null; + } + }); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs-int/src/test/resources/hdfs-sentry.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs-int/src/test/resources/hdfs-sentry.xml b/sentry-hdfs-int/src/test/resources/hdfs-sentry.xml new file mode 100644 index 0000000..511bfdd --- /dev/null +++ b/sentry-hdfs-int/src/test/resources/hdfs-sentry.xml @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>sentry.hdfs-plugin.path-prefixes</name> + <value>/user/hive/dw</value> + </property> + <property> + <name>sentry.hdfs-plugin.sentry-uri</name> + <value>thrift://localhost:1234</value> + </property> + <property> + <name>sentry.hdfs-plugin.stale-threshold.ms</name> + <value>-1</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/pom.xml b/sentry-hdfs/pom.xml new file mode 100644 index 0000000..5114c18 --- /dev/null +++ b/sentry-hdfs/pom.xml @@ -0,0 +1,66 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry</artifactId> + <version>1.5.0-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <artifactId>sentry-hdfs</artifactId> + <name>Sentry HDFS Integration</name> + + <dependencies> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>2.5.0</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-service-client</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>2.5.0</version> + <scope>provided</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPaths.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPaths.java new file mode 100644 index 0000000..9ea50c7 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPaths.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +public interface AuthzPaths { + + public boolean isUnderPrefix(String[] pathElements); + + public String findAuthzObject(String[] pathElements); + + public String findAuthzObjectExactMatch(String[] pathElements); + + public AuthzPathsDumper<? extends AuthzPaths> getPathsDump(); + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPathsDumper.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPathsDumper.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPathsDumper.java new file mode 100644 index 0000000..924d3b4 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPathsDumper.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import org.apache.sentry.provider.db.service.thrift.TPathsDump; + +public interface AuthzPathsDumper<K extends AuthzPaths> { + + public TPathsDump createPathsDump(); + + public K initializeFromDump(TPathsDump pathsDump); + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPermissions.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPermissions.java new file mode 100644 index 0000000..1631ae5 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/AuthzPermissions.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import org.apache.hadoop.fs.permission.AclEntry; + +import java.util.List; + +public interface AuthzPermissions { + + public List<AclEntry> getAcls(String authzObj); + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/ExtendedMetastoreClient.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/ExtendedMetastoreClient.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/ExtendedMetastoreClient.java new file mode 100644 index 0000000..c0358f4 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/ExtendedMetastoreClient.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ExtendedMetastoreClient implements MetastoreClient { + + private static Logger LOG = LoggerFactory.getLogger(ExtendedMetastoreClient.class); + + private HiveMetaStoreClient client; + private final HiveConf hiveConf; + public ExtendedMetastoreClient(HiveConf hiveConf) { + this.hiveConf = hiveConf; + } + + @Override + public List<Database> getAllDatabases() { + List<Database> retList = new ArrayList<Database>(); + HiveMetaStoreClient client = getClient(); + if (client != null) { + try { + for (String dbName : client.getAllDatabases()) { + retList.add(client.getDatabase(dbName)); + } + } catch (Exception e) { + LOG.error("Could not get All Databases !!", e); + } + } + return retList; + } + + @Override + public List<Table> getAllTablesOfDatabase(Database db) { + List<Table> retList = new ArrayList<Table>(); + HiveMetaStoreClient client = getClient(); + if (client != null) { + try { + for (String tblName : client.getAllTables(db.getName())) { + retList.add(client.getTable(db.getName(), tblName)); + } + } catch (Exception e) { + LOG.error(String.format( + "Could not get Tables for '%s' !!", db.getName()), e); + } + } + return retList; + } + + @Override + public List<Partition> listAllPartitions(Database db, Table tbl) { + HiveMetaStoreClient client = getClient(); + if (client != null) { + try { + return client.listPartitions(db.getName(), tbl.getTableName(), Short.MAX_VALUE); + } catch (Exception e) { + LOG.error(String.format( + "Could not get partitions for '%s'.'%s' !!", db.getName(), + tbl.getTableName()), e); + } + } + return new LinkedList<Partition>(); + } + + private HiveMetaStoreClient getClient() { + if (client == null) { + try { + client = new HiveMetaStoreClient(hiveConf); + return client; + } catch (MetaException e) { + client = null; + LOG.error("Could not create metastore client !!", e); + return null; + } + } else { + return client; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPaths.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPaths.java new file mode 100644 index 0000000..e445634 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPaths.java @@ -0,0 +1,467 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.fs.Path; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +public class HMSPaths implements AuthzPaths { + + @VisibleForTesting + static List<String> getPathElements(String path) { + path = path.trim(); + if (path.charAt(0) != Path.SEPARATOR_CHAR) { + throw new IllegalArgumentException("It must be an absolute path: " + + path); + } + List<String> list = new ArrayList<String>(32); + int idx = 0; + int found = path.indexOf(Path.SEPARATOR_CHAR, idx); + while (found > -1) { + if (found > idx) { + list.add(path.substring(idx, found)); + } + idx = found + 1; + found = path.indexOf(Path.SEPARATOR_CHAR, idx); + } + if (idx < path.length()) { + list.add(path.substring(idx)); + } + return list; + } + + @VisibleForTesting + static List<List<String>> gePathsElements(List<String> paths) { + List<List<String>> pathsElements = new ArrayList<List<String>>(paths.size()); + for (String path : paths) { + pathsElements.add(getPathElements(path)); + } + return pathsElements; + } + + @VisibleForTesting + enum EntryType { + DIR(true), + PREFIX(false), + AUTHZ_OBJECT(false); + + private boolean removeIfDangling; + + private EntryType(boolean removeIfDangling) { + this.removeIfDangling = removeIfDangling; + } + + public boolean isRemoveIfDangling() { + return removeIfDangling; + } + + public byte getByte() { + return (byte)toString().charAt(0); + } + + public static EntryType fromByte(byte b) { + switch (b) { + case ((byte)'D'): + return DIR; + case ((byte)'P'): + return PREFIX; + case ((byte)'A'): + return AUTHZ_OBJECT; + default: + return null; + } + } + } + + @VisibleForTesting + static class Entry { + private Entry parent; + private EntryType type; + private final String pathElement; + private String authzObj; + private final Map<String, Entry> children; + + Entry(Entry parent, String pathElement, EntryType type, + String authzObj) { + this.parent = parent; + this.type = type; + this.pathElement = pathElement; + this.authzObj = authzObj; + children = new HashMap<String, Entry>(); + } + + private void setAuthzObj(String authzObj) { + this.authzObj = authzObj; + } + + private void setType(EntryType type) { + this.type = type; + } + + protected void removeParent() { + parent = null; + } + + public String toString() { + return String.format("Entry[fullPath: %s, type: %s, authObject: %s]", + getFullPath(), type, authzObj); + } + + private Entry createChild(List<String> pathElements, EntryType type, + String authzObj) { + Entry entryParent = this; + for (int i = 0; i < pathElements.size() - 1; i++) { + String pathElement = pathElements.get(i); + Entry child = entryParent.getChildren().get(pathElement); + if (child == null) { + child = new Entry(entryParent, pathElement, EntryType.DIR, null); + entryParent.getChildren().put(pathElement, child); + } + entryParent = child; + } + String lastPathElement = pathElements.get(pathElements.size() - 1); + Entry child = entryParent.getChildren().get(lastPathElement); + if (child == null) { + child = new Entry(entryParent, lastPathElement, type, authzObj); + entryParent.getChildren().put(lastPathElement, child); + } else if (type == EntryType.AUTHZ_OBJECT && + child.getType() == EntryType.DIR) { + // if the entry already existed as dir, we change it to be a authz obj + child.setAuthzObj(authzObj); + child.setType(EntryType.AUTHZ_OBJECT); + } + return child; + } + + public static Entry createRoot(boolean asPrefix) { + return new Entry(null, "/", (asPrefix) + ? EntryType.PREFIX : EntryType.DIR, null); + } + + private String toPath(List<String> arr) { + StringBuilder sb = new StringBuilder(); + for (String s : arr) { + sb.append(Path.SEPARATOR).append(s); + } + return sb.toString(); + } + + public Entry createPrefix(List<String> pathElements) { + Entry prefix = findPrefixEntry(pathElements); + if (prefix != null) { + throw new IllegalArgumentException(String.format( + "Cannot add prefix '%s' under an existing prefix '%s'", + toPath(pathElements), prefix.getFullPath())); + } + return createChild(pathElements, EntryType.PREFIX, null); + } + + public Entry createAuthzObjPath(List<String> pathElements, String authzObj) { + Entry entry = null; + Entry prefix = findPrefixEntry(pathElements); + if (prefix != null) { + // we only create the entry if is under a prefix, else we ignore it + entry = createChild(pathElements, EntryType.AUTHZ_OBJECT, authzObj); + } + return entry; + } + + public void delete() { + if (getParent() != null) { + if (getChildren().isEmpty()) { + getParent().getChildren().remove(getPathElement()); + getParent().deleteIfDangling(); + parent = null; + } else { + // if the entry was for an authz object and has children, we + // change it to be a dir entry. + if (getType() == EntryType.AUTHZ_OBJECT) { + setType(EntryType.DIR); + setAuthzObj(null); + } + } + } + } + + private void deleteIfDangling() { + if (getChildren().isEmpty() && getType().isRemoveIfDangling()) { + delete(); + } + } + + public Entry getParent() { + return parent; + } + + public EntryType getType() { + return type; + } + + public String getPathElement() { + return pathElement; + } + + public String getAuthzObj() { + return authzObj; + } + + @SuppressWarnings("unchecked") + public Map<String, Entry> getChildren() { + return children; + } + + public Entry findPrefixEntry(List<String> pathElements) { + Preconditions.checkArgument(pathElements != null, + "pathElements cannot be NULL"); + return (getType() == EntryType.PREFIX) + ? this : findPrefixEntry(pathElements, 0); + } + + private Entry findPrefixEntry(List<String> pathElements, int index) { + Entry prefixEntry = null; + if (index == pathElements.size()) { + prefixEntry = null; + } else { + Entry child = getChildren().get(pathElements.get(index)); + if (child != null) { + if (child.getType() == EntryType.PREFIX) { + prefixEntry = child; + } else { + prefixEntry = child.findPrefixEntry(pathElements, index + 1); + } + } + } + return prefixEntry; + } + + public Entry find(String[] pathElements, boolean isPartialMatchOk) { + Preconditions.checkArgument( + pathElements != null && pathElements.length > 0, + "pathElements cannot be NULL or empty"); + return find(pathElements, 0, isPartialMatchOk, null); + } + + private Entry find(String[] pathElements, int index, + boolean isPartialMatchOk, Entry lastAuthObj) { + Entry found = null; + if (index == pathElements.length) { + if (isPartialMatchOk && (getType() == EntryType.AUTHZ_OBJECT)) { + found = this; + } + } else { + Entry child = getChildren().get(pathElements[index]); + if (child != null) { + if (index == pathElements.length - 1) { + found = (child.getType() == EntryType.AUTHZ_OBJECT) ? child : lastAuthObj; + } else { + found = child.find(pathElements, index + 1, isPartialMatchOk, + (child.getType() == EntryType.AUTHZ_OBJECT) ? child : lastAuthObj); + } + } else { + if (isPartialMatchOk) { + found = lastAuthObj; + } + } + } + return found; + } + + public String getFullPath() { + String path = getFullPath(this, new StringBuilder()).toString(); + if (path.isEmpty()) { + path = Path.SEPARATOR; + } + return path; + } + + private StringBuilder getFullPath(Entry entry, StringBuilder sb) { + if (entry.getParent() != null) { + getFullPath(entry.getParent(), sb).append(Path.SEPARATOR).append( + entry.getPathElement()); + } + return sb; + } + + } + + private volatile Entry root; + private Map<String, Set<Entry>> authzObjToPath; + + public HMSPaths(String[] pathPrefixes) { + boolean rootPrefix = false; + for (String pathPrefix : pathPrefixes) { + rootPrefix = rootPrefix || pathPrefix.equals(Path.SEPARATOR); + } + if (rootPrefix && pathPrefixes.length > 1) { + throw new IllegalArgumentException( + "Root is a path prefix, there cannot be other path prefixes"); + } + root = Entry.createRoot(rootPrefix); + if (!rootPrefix) { + for (String pathPrefix : pathPrefixes) { + root.createPrefix(getPathElements(pathPrefix)); + } + } + authzObjToPath = new HashMap<String, Set<Entry>>(); + } + + HMSPaths() { + authzObjToPath = new HashMap<String, Set<Entry>>(); + } + + void _addAuthzObject(String authzObj, List<String> authzObjPaths) { + addAuthzObject(authzObj, gePathsElements(authzObjPaths)); + } + + void addAuthzObject(String authzObj, List<List<String>> authzObjPathElements) { + Set<Entry> previousEntries = authzObjToPath.get(authzObj); + Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size()); + for (List<String> pathElements : authzObjPathElements) { + Entry e = root.createAuthzObjPath(pathElements, authzObj); + if (e != null) { + newEntries.add(e); + } else { + // LOG WARN IGNORING PATH, no prefix + } + } + authzObjToPath.put(authzObj, newEntries); + if (previousEntries != null) { + previousEntries.removeAll(newEntries); + if (!previousEntries.isEmpty()) { + for (Entry entry : previousEntries) { + entry.delete(); + } + } + } + } + + void addPathsToAuthzObject(String authzObj, + List<List<String>> authzObjPathElements, boolean createNew) { + Set<Entry> entries = authzObjToPath.get(authzObj); + if (entries != null) { + Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size()); + for (List<String> pathElements : authzObjPathElements) { + Entry e = root.createAuthzObjPath(pathElements, authzObj); + if (e != null) { + newEntries.add(e); + } else { + // LOG WARN IGNORING PATH, no prefix + } + } + entries.addAll(newEntries); + } else { + if (createNew) { + addAuthzObject(authzObj, authzObjPathElements); + } + // LOG WARN object does not exist + } + } + + void _addPathsToAuthzObject(String authzObj, List<String> authzObjPaths) { + addPathsToAuthzObject(authzObj, gePathsElements(authzObjPaths), false); + } + + void addPathsToAuthzObject(String authzObj, List<List<String>> authzObjPaths) { + addPathsToAuthzObject(authzObj, authzObjPaths, false); + } + + void deletePathsFromAuthzObject(String authzObj, + List<List<String>> authzObjPathElements) { + Set<Entry> entries = authzObjToPath.get(authzObj); + if (entries != null) { + Set<Entry> toDelEntries = new HashSet<Entry>(authzObjPathElements.size()); + for (List<String> pathElements : authzObjPathElements) { + Entry entry = root.find( + pathElements.toArray(new String[pathElements.size()]), false); + if (entry != null) { + entry.delete(); + toDelEntries.add(entry); + } else { + // LOG WARN IGNORING PATH, it was not in registered + } + } + entries.removeAll(toDelEntries); + } else { + // LOG WARN object does not exist + } + } + + void deleteAuthzObject(String authzObj) { + Set<Entry> entries = authzObjToPath.remove(authzObj); + if (entries != null) { + for (Entry entry : entries) { + entry.delete(); + } + } + } + + @Override + public String findAuthzObject(String[] pathElements) { + return findAuthzObject(pathElements, true); + } + + @Override + public String findAuthzObjectExactMatch(String[] pathElements) { + return findAuthzObject(pathElements, false); + } + + public String findAuthzObject(String[] pathElements, boolean isPartialOk) { + // Handle '/' + if ((pathElements == null)||(pathElements.length == 0)) return null; + String authzObj = null; + Entry entry = root.find(pathElements, isPartialOk); + if (entry != null) { + authzObj = entry.getAuthzObj(); + } + return authzObj; + } + + @Override + public boolean isUnderPrefix(String[] pathElements) { + return root.findPrefixEntry(Lists.newArrayList(pathElements)) != null; + } + + // Used by the serializer + Entry getRootEntry() { + return root; + } + + void setRootEntry(Entry root) { + this.root = root; + } + + void setAuthzObjToPathMapping(Map<String, Set<Entry>> mapping) { + authzObjToPath = mapping; + } + + @Override + public HMSPathsSerDe getPathsDump() { + return new HMSPathsSerDe(this); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPathsSerDe.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPathsSerDe.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPathsSerDe.java new file mode 100644 index 0000000..7a25d29 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/HMSPathsSerDe.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.sentry.hdfs.HMSPaths.Entry; +import org.apache.sentry.hdfs.HMSPaths.EntryType; +import org.apache.sentry.provider.db.service.thrift.TPathsDump; +import org.apache.sentry.provider.db.service.thrift.TPathEntry; + +public class HMSPathsSerDe implements AuthzPathsDumper<HMSPaths> { + + private final HMSPaths hmsPaths; + + static class Tuple { + final TPathEntry entry; + final int id; + Tuple(TPathEntry entry, int id) { + this.entry = entry; + this.id = id; + } + } + + public HMSPathsSerDe(HMSPaths hmsPaths) { + this.hmsPaths = hmsPaths; + } + + @Override + public TPathsDump createPathsDump() { + AtomicInteger counter = new AtomicInteger(0); + Map<Integer, TPathEntry> idMap = new HashMap<Integer, TPathEntry>(); + Tuple tRootTuple = + createTPathEntry(hmsPaths.getRootEntry(), counter, idMap); + idMap.put(tRootTuple.id, tRootTuple.entry); + cloneToTPathEntry(hmsPaths.getRootEntry(), tRootTuple.entry, counter, idMap); + return new TPathsDump(tRootTuple.id, idMap); + } + + private void cloneToTPathEntry(Entry parent, TPathEntry tParent, + AtomicInteger counter, Map<Integer, TPathEntry> idMap) { + for (Entry child : parent.getChildren().values()) { + Tuple childTuple = createTPathEntry(child, counter, idMap); + tParent.getChildren().add(childTuple.id); + cloneToTPathEntry(child, childTuple.entry, counter, idMap); + } + } + + private Tuple createTPathEntry(Entry entry, AtomicInteger idCounter, + Map<Integer, TPathEntry> idMap) { + int myId = idCounter.incrementAndGet(); + TPathEntry tEntry = new TPathEntry(entry.getType().getByte(), + entry.getPathElement(), new HashSet<Integer>()); + if (entry.getAuthzObj() != null) { + tEntry.setAuthzObj(entry.getAuthzObj()); + } + idMap.put(myId, tEntry); + return new Tuple(tEntry, myId); + } + + @Override + public HMSPaths initializeFromDump(TPathsDump pathDump) { + HMSPaths hmsPaths = new HMSPaths(); + TPathEntry tRootEntry = pathDump.getNodeMap().get(pathDump.getRootId()); + Entry rootEntry = new Entry(null, tRootEntry.getPathElement(), + EntryType.fromByte(tRootEntry.getType()), tRootEntry.getAuthzObj()); + Map<String, Set<Entry>> authzObjToPath = new HashMap<String, Set<Entry>>(); + cloneToEntry(tRootEntry, rootEntry, pathDump.getNodeMap(), authzObjToPath); + hmsPaths.setRootEntry(rootEntry); + hmsPaths.setAuthzObjToPathMapping(authzObjToPath); + return hmsPaths; + } + + private void cloneToEntry(TPathEntry tParent, Entry parent, + Map<Integer, TPathEntry> idMap, Map<String, Set<Entry>> authzObjToPath) { + for (Integer id : tParent.getChildren()) { + TPathEntry tChild = idMap.get(id); + Entry child = new Entry(parent, tChild.getPathElement(), + EntryType.fromByte(tChild.getType()), tChild.getAuthzObj()); + if (child.getAuthzObj() != null) { + Set<Entry> paths = authzObjToPath.get(child.getAuthzObj()); + if (paths == null) { + paths = new HashSet<Entry>(); + authzObjToPath.put(child.getAuthzObj(), paths); + } + paths.add(child); + } + parent.getChildren().put(child.getPathElement(), child); + cloneToEntry(tChild, child, idMap, authzObjToPath); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/MetastoreClient.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/MetastoreClient.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/MetastoreClient.java new file mode 100644 index 0000000..3b64756 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/MetastoreClient.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; + +public interface MetastoreClient { + + public List<Database> getAllDatabases(); + + public List<Table> getAllTablesOfDatabase(Database db); + + public List<Partition> listAllPartitions(Database db, Table tbl); + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java new file mode 100644 index 0000000..faa28f1 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.LinkedList; +import java.util.List; + +import org.apache.sentry.provider.db.service.thrift.TPathChanges; +import org.apache.sentry.provider.db.service.thrift.TPathsUpdate; + +import com.google.common.collect.Lists; + +public class PathsUpdate implements Updateable.Update { + + public static String ALL_PATHS = "__ALL_PATHS__"; + + private final TPathsUpdate tPathsUpdate; + + public PathsUpdate(TPathsUpdate tPathsUpdate) { + this.tPathsUpdate = tPathsUpdate; + } + + public PathsUpdate(long seqNum, boolean hasFullImage) { + tPathsUpdate = new TPathsUpdate(hasFullImage, seqNum, + new LinkedList<TPathChanges>()); + } + + @Override + public boolean hasFullImage() { + return tPathsUpdate.isHasFullImage(); + } + public TPathChanges newPathChange(String authzObject) { + TPathChanges pathChanges = new TPathChanges(authzObject, + new LinkedList<List<String>>(), new LinkedList<List<String>>()); + tPathsUpdate.addToPathChanges(pathChanges); + return pathChanges; + } + public List<TPathChanges> getPathChanges() { + return tPathsUpdate.getPathChanges(); + } + + @Override + public long getSeqNum() { + return tPathsUpdate.getSeqNum(); + } + + @Override + public void setSeqNum(long seqNum) { + tPathsUpdate.setSeqNum(seqNum); + } + + public TPathsUpdate getThriftObject() { + return tPathsUpdate; + } + + + + public static List<String> cleanPath(String path) { + try { + return Lists.newArrayList(new URI(path).getPath().split("^/")[1] + .split("/")); + } catch (URISyntaxException e) { + throw new RuntimeException("Incomprehensible path [" + path + "]"); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java new file mode 100644 index 0000000..d9a6592 --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; + +import org.apache.sentry.provider.db.service.thrift.TPermissionsUpdate; +import org.apache.sentry.provider.db.service.thrift.TPrivilegeChanges; +import org.apache.sentry.provider.db.service.thrift.TRoleChanges; + +public class PermissionsUpdate implements Updateable.Update { + + public static String ALL_AUTHZ_OBJ = "__ALL_AUTHZ_OBJ__"; + public static String ALL_PRIVS = "__ALL_PRIVS__"; + public static String ALL_ROLES = "__ALL_ROLES__"; + public static String ALL_GROUPS = "__ALL_GROUPS__"; + + private final TPermissionsUpdate tPermUpdate; + + public PermissionsUpdate(TPermissionsUpdate tPermUpdate) { + this.tPermUpdate = tPermUpdate; + } + + public PermissionsUpdate(long seqNum, boolean hasFullImage) { + this.tPermUpdate = new TPermissionsUpdate(hasFullImage, seqNum, + new HashMap<String, TPrivilegeChanges>(), + new HashMap<String, TRoleChanges>()); + } + + @Override + public long getSeqNum() { + return tPermUpdate.getSeqNum(); + } + + @Override + public void setSeqNum(long seqNum) { + tPermUpdate.setSeqNum(seqNum); + } + + @Override + public boolean hasFullImage() { + return tPermUpdate.isHasfullImage(); + } + + public TPrivilegeChanges addPrivilegeUpdate(String authzObj) { + if (tPermUpdate.getPrivilegeChanges().containsKey(authzObj)) { + return tPermUpdate.getPrivilegeChanges().get(authzObj); + } + TPrivilegeChanges privUpdate = new TPrivilegeChanges(authzObj, + new HashMap<String, String>(), new HashMap<String, String>()); + tPermUpdate.getPrivilegeChanges().put(authzObj, privUpdate); + return privUpdate; + } + + public TRoleChanges addRoleUpdate(String role) { + if (tPermUpdate.getRoleChanges().containsKey(role)) { + return tPermUpdate.getRoleChanges().get(role); + } + TRoleChanges roleUpdate = new TRoleChanges(role, new LinkedList<String>(), + new LinkedList<String>()); + tPermUpdate.getRoleChanges().put(role, roleUpdate); + return roleUpdate; + } + + public Collection<TRoleChanges> getRoleUpdates() { + return tPermUpdate.getRoleChanges().values(); + } + + public Collection<TPrivilegeChanges> getPrivilegeUpdates() { + return tPermUpdate.getPrivilegeChanges().values(); + } + + public TPermissionsUpdate getThriftObject() { + return tPermUpdate; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/Updateable.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/Updateable.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/Updateable.java new file mode 100644 index 0000000..1649ffc --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/Updateable.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.concurrent.locks.ReadWriteLock; + +public interface Updateable<K extends Updateable.Update> { + + public interface Update { + + boolean hasFullImage(); + + long getSeqNum(); + + void setSeqNum(long seqNum); + + } + + /** + * Apply multiple partial updates in order + * @param update + * @param lock External Lock. + * @return + */ + public void updatePartial(Iterable<K> update, ReadWriteLock lock); + + /** + * This returns a new object with the full update applied + * @param update + * @return + */ + public Updateable<K> updateFull(K update); + + /** + * Return sequence number of Last Update + */ + public long getLastUpdatedSeqNum(); + + /** + * Create and Full image update of the local data structure + * @param currSeqNum + * @return + */ + public K createFullImageUpdate(long currSeqNum); + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java new file mode 100644 index 0000000..8680d5d --- /dev/null +++ b/sentry-hdfs/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; + +import org.apache.sentry.provider.db.service.thrift.TPathChanges; +import org.apache.sentry.provider.db.service.thrift.TPathsDump; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UpdateableAuthzPaths implements AuthzPaths, Updateable<PathsUpdate> { + private volatile HMSPaths paths; + private final AtomicLong seqNum = new AtomicLong(0); + + private static Logger LOG = LoggerFactory.getLogger(UpdateableAuthzPaths.class); + + public UpdateableAuthzPaths(String[] pathPrefixes) { + this.paths = new HMSPaths(pathPrefixes); + } + + UpdateableAuthzPaths(HMSPaths paths) { + this.paths = paths; + } + + @Override + public boolean isUnderPrefix(String[] pathElements) { + return paths.isUnderPrefix(pathElements); + } + + @Override + public String findAuthzObject(String[] pathElements) { + return paths.findAuthzObject(pathElements); + } + + @Override + public String findAuthzObjectExactMatch(String[] pathElements) { + return paths.findAuthzObjectExactMatch(pathElements); + } + + @Override + public UpdateableAuthzPaths updateFull(PathsUpdate update) { + UpdateableAuthzPaths other = getPathsDump().initializeFromDump( + update.getThriftObject().getPathsDump()); + other.seqNum.set(update.getSeqNum()); + return other; + } + + @Override + public void updatePartial(Iterable<PathsUpdate> updates, ReadWriteLock lock) { + lock.writeLock().lock(); + try { + int counter = 0; + for (PathsUpdate update : updates) { + applyPartialUpdate(update); + if (++counter > 99) { + counter = 0; + lock.writeLock().unlock(); + lock.writeLock().lock(); + } + seqNum.set(update.getSeqNum()); + LOG.warn("##### Updated paths seq Num [" + seqNum.get() + "]"); + } + } finally { + lock.writeLock().unlock(); + } + } + + private void applyPartialUpdate(PathsUpdate update) { + for (TPathChanges pathChanges : update.getPathChanges()) { + paths.addPathsToAuthzObject(pathChanges.getAuthzObj(), pathChanges + .getAddPaths(), true); + List<List<String>> delPaths = pathChanges.getDelPaths(); + if ((delPaths.size() == 1) && (delPaths.get(0).size() == 1) + && (delPaths.get(0).get(0).equals(PathsUpdate.ALL_PATHS))) { + // Remove all paths.. eg. drop table + paths.deleteAuthzObject(pathChanges.getAuthzObj()); + } else { + paths.deletePathsFromAuthzObject(pathChanges.getAuthzObj(), pathChanges + .getDelPaths()); + } + } + } + + @Override + public long getLastUpdatedSeqNum() { + return seqNum.get(); + } + + @Override + public PathsUpdate createFullImageUpdate(long currSeqNum) { + PathsUpdate pathsUpdate = new PathsUpdate(currSeqNum, true); + pathsUpdate.getThriftObject().setPathsDump(getPathsDump().createPathsDump()); + return pathsUpdate; + } + + @Override + public AuthzPathsDumper<UpdateableAuthzPaths> getPathsDump() { + return new AuthzPathsDumper<UpdateableAuthzPaths>() { + + @Override + public TPathsDump createPathsDump() { + return UpdateableAuthzPaths.this.paths.getPathsDump().createPathsDump(); + } + + @Override + public UpdateableAuthzPaths initializeFromDump(TPathsDump pathsDump) { + return new UpdateableAuthzPaths(new HMSPaths().getPathsDump().initializeFromDump( + pathsDump)); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAdapter.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAdapter.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAdapter.java new file mode 100644 index 0000000..24c63a5 --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAdapter.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +//import org.apache.sentry.provider.db.service.thrift.UpdateForwarder; + +public class DummyAdapter { +//public class DummyAdapter<K extends UpdateForwarder.Update> { +// +// private final UpdateForwarder<K> destCache; +// private final UpdateForwarder<K> srcCache; +// +// public DummyAdapter(UpdateForwarder<K> destCache, UpdateForwarder<K> srcCache) { +// super(); +// this.destCache = destCache; +// this.srcCache = srcCache; +// } +// +// public void getDestToPullUpdatesFromSrc() { +// for (K update : srcCache.getAllUpdatesFrom(destCache.getLastCommitted() + 1)) { +// destCache.handleUpdateNotification(update); +// } +// } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAuthzSource.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAuthzSource.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAuthzSource.java new file mode 100644 index 0000000..57299c8 --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAuthzSource.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +//import org.apache.sentry.hdfs.old.AuthzPermCache.AuthzSource; +//import org.apache.sentry.hdfs.old.AuthzPermCache.PrivilegeInfo; +//import org.apache.sentry.hdfs.old.AuthzPermCache.RoleInfo; + +public class DummyAuthzSource { +//public class DummyAuthzSource implements AuthzSource{ +// +// public Map<String, PrivilegeInfo> privs = new HashMap<String, PrivilegeInfo>(); +// public Map<String, RoleInfo> roles = new HashMap<String, RoleInfo>(); +// +// @Override +// public PrivilegeInfo loadPrivilege(String authzObj) throws Exception { +// return privs.get(authzObj); +// } +// +// @Override +// public RoleInfo loadGroupsForRole(String group) throws Exception { +// return roles.get(group); +// } +// +// @Override +// public PermissionsUpdate createFullImage(long seqNum) { +// PermissionsUpdate retVal = new PermissionsUpdate(seqNum, true); +// for (Map.Entry<String, PrivilegeInfo> pE : privs.entrySet()) { +// PrivilegeChanges pUpdate = retVal.addPrivilegeUpdate(pE.getKey()); +// PrivilegeInfo pInfo = pE.getValue(); +// for (Map.Entry<String, FsAction> ent : pInfo.roleToPermission.entrySet()) { +// pUpdate.addPrivilege(ent.getKey(), ent.getValue().SYMBOL); +// } +// } +// for (Map.Entry<String, RoleInfo> rE : roles.entrySet()) { +// RoleChanges rUpdate = retVal.addRoleUpdate(rE.getKey()); +// RoleInfo rInfo = rE.getValue(); +// for (String role : rInfo.groups) { +// rUpdate.addGroup(role); +// } +// } +// return retVal; +// } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyHMSClient.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyHMSClient.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyHMSClient.java new file mode 100644 index 0000000..3f66c87 --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyHMSClient.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.sentry.hdfs.MetastoreClient; + +public class DummyHMSClient implements MetastoreClient { + + private HashMap<Database, HashMap<Table, HashSet<Partition>>> hmsData = + new HashMap<Database, HashMap<Table, HashSet<Partition>>>(); + + @Override + public List<Database> getAllDatabases() { + return new ArrayList<Database>(hmsData.keySet()); + } + + @Override + public List<Table> getAllTablesOfDatabase(Database db) { + if (hmsData.containsKey(db)) { + return new ArrayList<Table>(hmsData.get(db).keySet()); + } + return new ArrayList<Table>(); + } + + @Override + public List<Partition> listAllPartitions(Database db, Table tbl) { + if (hmsData.containsKey(db)) { + if (hmsData.get(db).containsKey(tbl)) { + return new ArrayList<Partition>(hmsData.get(db).get(tbl)); + } + } + return new ArrayList<Partition>(); + } + + public Database addDb(String dbName, String location) { + Database db = new Database(dbName, null, location, null); + hmsData.put(db, new HashMap<Table, HashSet<Partition>>()); + return db; + } + + public Table addTable(Database db, String tblName, String location) { + Table tbl = + new Table(tblName, db.getName(), null, 0, 0, 0, + new StorageDescriptor(null, location, null, null, false, 0, null, null, null, null), + null, null, null, null, null); + hmsData.get(db).put(tbl, new HashSet<Partition>()); + return tbl; + } + + public void addPartition(Database db, Table tbl, String partitionPath) { + Partition part = new Partition(null, db.getName(), tbl.getTableName(), 0, 0, + new StorageDescriptor(null, partitionPath, null, null, false, 0, null, null, null, null), null); + hmsData.get(db).get(tbl).add(part); + } +}
