Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 5630fc5ce -> e7ec257a7
SENTRY-1325: Store HMSPaths in Sentry DB to allow fast failover (Hao Hao, Reviewed by Colin Patrick McCabe and Anne Yu) Change-Id: Idc4bb72436b6b4be098496ef074a05a9390ae135 Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/e7ec257a Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/e7ec257a Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/e7ec257a Branch: refs/heads/sentry-ha-redesign Commit: e7ec257a76c7175a49e95b17ca4676dbd74a8d67 Parents: 5630fc5 Author: hahao <[email protected]> Authored: Fri Jun 24 15:18:24 2016 -0700 Committer: hahao <[email protected]> Committed: Fri Jun 24 15:18:24 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/sentry/hdfs/HMSPaths.java | 93 +++++++++++++---- .../org/apache/sentry/hdfs/HMSPathsDumper.java | 2 +- .../db/service/model/MAuthzPathsMapping.java | 102 +++++++++++++++++++ .../provider/db/service/model/package.jdo | 25 +++++ .../db/service/persistent/SentryStore.java | 71 +++++++++++++ .../db/service/persistent/TestSentryStore.java | 12 +++ 6 files changed, 287 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java index 6d2ab23..e4850f4 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java @@ -116,7 +116,7 @@ public class HMSPaths implements AuthzPaths { // Path of child element to the path entry mapping. // e.g. 'b' -> '/a/b' - private final Map<String, Entry> children; + private final Map<String, Entry> children = new HashMap<String, Entry>(); Entry(Entry parent, String pathElement, EntryType type, String authzObj) { @@ -125,7 +125,6 @@ public class HMSPaths implements AuthzPaths { this.pathElement = pathElement; this.authzObjs = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER); addAuthzObj(authzObj); - children = new HashMap<String, Entry>(); } Entry(Entry parent, String pathElement, EntryType type, @@ -135,7 +134,6 @@ public class HMSPaths implements AuthzPaths { this.pathElement = pathElement; this.authzObjs = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER); addAuthzObjs(authzObjs); - children = new HashMap<String, Entry>(); } // Get all the mapping of the children element to @@ -179,6 +177,66 @@ public class HMSPaths implements AuthzPaths { getFullPath(), type, Joiner.on(",").join(authzObjs)); } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + Entry other = (Entry) obj; + if (parent == null) { + if (other.parent != null) { + return false; + } + } else if (!parent.equals(other.parent)) { + return false; + } + + if (type == null) { + if (other.type != null) { + return false; + } + } else if (!type.equals(other.type)) { + return false; + } + + if (pathElement == null) { + if (other.pathElement != null) { + return false; + } + } else if (!pathElement.equals(other.pathElement)) { + return false; + } + + if (authzObjs == null) { + if (other.authzObjs != null) { + return false; + } + } else if (!authzObjs.equals(other.authzObjs)) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((parent == null) ? 0 : parent.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + result = prime * result + ((pathElement == null) ? 0 : pathElement.hashCode()); + result = prime * result + ((authzObjs == null) ? 0 : authzObjs.hashCode()); + + return result; + } + /** * Create a child entry based on the path, type and authzObj that * associates with it. @@ -332,12 +390,10 @@ public class HMSPaths implements AuthzPaths { return authzObjs; } - - public Entry findPrefixEntry(List<String> pathElements) { Preconditions.checkArgument(pathElements != null, "pathElements cannot be NULL"); - return (getType() == EntryType.PREFIX) + return (getType() == EntryType.PREFIX) ? this : findPrefixEntry(pathElements, 0); } @@ -413,7 +469,10 @@ public class HMSPaths implements AuthzPaths { // The hive authorized objects to path entries mapping. // One authorized object can map to a set of path entries. - private Map<String, Set<Entry>> authzObjToPath; + private Map<String, Set<Entry>> authzObjToEntries; + + public HMSPaths() { + } public HMSPaths(String[] pathPrefixes) { boolean rootPrefix = false; @@ -433,7 +492,7 @@ public class HMSPaths implements AuthzPaths { } } - authzObjToPath = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER); + authzObjToEntries = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER); } void _addAuthzObject(String authzObj, List<String> authzObjPaths) { @@ -441,7 +500,7 @@ public class HMSPaths implements AuthzPaths { } void addAuthzObject(String authzObj, List<List<String>> authzObjPathElements) { - Set<Entry> previousEntries = authzObjToPath.get(authzObj); + Set<Entry> previousEntries = authzObjToEntries.get(authzObj); Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size()); for (List<String> pathElements : authzObjPathElements) { Entry e = root.createAuthzObjPath(pathElements, authzObj); @@ -451,7 +510,7 @@ public class HMSPaths implements AuthzPaths { LOG.warn("Ignoring path, no prefix"); } } - authzObjToPath.put(authzObj, newEntries); + authzObjToEntries.put(authzObj, newEntries); if (previousEntries != null) { previousEntries.removeAll(newEntries); if (!previousEntries.isEmpty()) { @@ -464,7 +523,7 @@ public class HMSPaths implements AuthzPaths { void addPathsToAuthzObject(String authzObj, List<List<String>> authzObjPathElements, boolean createNew) { - Set<Entry> entries = authzObjToPath.get(authzObj); + Set<Entry> entries = authzObjToEntries.get(authzObj); if (entries != null) { Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size()); for (List<String> pathElements : authzObjPathElements) { @@ -501,7 +560,7 @@ public class HMSPaths implements AuthzPaths { */ void deletePathsFromAuthzObject(String authzObj, List<List<String>> authzObjPathElements) { - Set<Entry> entries = authzObjToPath.get(authzObj); + Set<Entry> entries = authzObjToEntries.get(authzObj); if (entries != null) { Set<Entry> toDelEntries = new HashSet<Entry>(authzObjPathElements.size()); for (List<String> pathElements : authzObjPathElements) { @@ -522,7 +581,7 @@ public class HMSPaths implements AuthzPaths { } void deleteAuthzObject(String authzObj) { - Set<Entry> entries = authzObjToPath.remove(authzObj); + Set<Entry> entries = authzObjToEntries.remove(authzObj); if (entries != null) { for (Entry entry : entries) { entry.deleteAuthzObject(authzObj); @@ -593,12 +652,12 @@ public class HMSPaths implements AuthzPaths { } } if(samePaths) { - Set<Entry> eSet = authzObjToPath.get(oldName); + Set<Entry> eSet = authzObjToEntries.get(oldName); if (eSet == null) { LOG.warn("Unexpected state in renameAuthzObject, cannot find oldName in authzObjToPath: oldName=" + oldName + " newName=" + newName + " oldPath=" + oldPathElems + " newPath=" + newPathElems); } else { - authzObjToPath.put(newName, eSet); + authzObjToEntries.put(newName, eSet); for (Entry e : eSet) { if (e.getAuthzObjs().contains(oldName)) { e.addAuthzObj(newName); @@ -632,8 +691,8 @@ public class HMSPaths implements AuthzPaths { this.root = root; } - void setAuthzObjToPathMapping(Map<String, Set<Entry>> mapping) { - authzObjToPath = mapping; + void setAuthzObjToEntryMapping(Map<String, Set<Entry>> mapping) { + authzObjToEntries = mapping; } @Override http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java index e759ff1..7e56952 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java @@ -86,7 +86,7 @@ public class HMSPathsDumper implements AuthzPathsDumper<HMSPaths> { cloneToEntry(tRootEntry, rootEntry, pathDump.getNodeMap(), authzObjToPath, rootEntry.getType() == EntryType.PREFIX); newHmsPaths.setRootEntry(rootEntry); - newHmsPaths.setAuthzObjToPathMapping(authzObjToPath); + newHmsPaths.setAuthzObjToEntryMapping(authzObjToPath); return newHmsPaths; } http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java new file mode 100644 index 0000000..56e456e --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.model; + +import javax.jdo.annotations.PersistenceCapable; +import java.util.Set; + +/** + * Transactional database backend for storing HMS Paths Updates. Any changes to this object + * require re-running the maven build so DN can re-enhance. + */ +@PersistenceCapable +public class MAuthzPathsMapping { + + private String authzObjName; + private Set<String> paths; + private long createTimeMs; + + public MAuthzPathsMapping(String authzObjName, Set<String> paths, long createTimeMs) { + this.authzObjName = authzObjName; + this.paths = paths; + this.createTimeMs = createTimeMs; + } + + public long getCreateTime() { + return createTimeMs; + } + + public void setCreateTime(long createTime) { + this.createTimeMs = createTime; + } + + public String getAuthzObjName() { + return authzObjName; + } + + public void setAuthzObjName(String authzObjName) { + this.authzObjName = authzObjName; + } + + public void setPaths(Set<String> paths) { + this.paths = paths; + } + + public Set<String> getPaths() { + return paths; + } + + @Override + public String toString() { + return "MSentryPathsUpdate [authzObj=" + authzObjName + "], paths=[" + paths.toString() + + "], createTimeMs=[" + String.valueOf(createTimeMs) + "]"; + } + + @Override + public int hashCode() { + return authzObjName == null ? 0 : authzObjName.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + MAuthzPathsMapping other = (MAuthzPathsMapping) obj; + + if (authzObjName == null) { + if (other.authzObjName != null) { + return false; + } + } else if (!authzObjName.equals(other.authzObjName)) { + return false; + } + + return true; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo index b3b9494..fb5470f 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo @@ -237,6 +237,31 @@ </field> </class> + <class name="MAuthzPathsMapping" identity-type="datastore" table="AUTHZ_PATHS_MAPPING" detachable="true"> + <datastore-identity> + <column name="AUTHZ_OBJ_ID"/> + </datastore-identity> + <!-- + authzObjName is composed by hive database name, and table name. e.g. "default.tb1". Since + both hive database name, and table name have restrictions to be at most 128 characters long, + 384 characters length should be enough for AUTHZ_OBJ_NAM. + --> + <field name="authzObjName"> + <column name="AUTHZ_OBJ_NAME" length="384" jdbc-type="VARCHAR"/> + <index name="AuthzObjName" unique="true"/> + </field> + <field name="createTimeMs"> + <column name="CREATE_TIME_MS" jdbc-type="BIGINT"/> + </field> + <field name = "paths"> + <collection element-type="java.lang.String"/> + <join/> + <element> + <column name="PATHS"/> + </element> + </field> + </class> + </package> </jdo> http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index b7ef0e9..7dad496 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -56,6 +56,7 @@ import org.apache.sentry.core.common.exception.SentryAlreadyExistsException; import org.apache.sentry.core.common.exception.SentryGrantDeniedException; import org.apache.sentry.core.common.exception.SentryInvalidInputException; import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; +import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping; import org.apache.sentry.provider.db.service.model.MSentryGroup; import org.apache.sentry.provider.db.service.model.MSentryPrivilege; import org.apache.sentry.provider.db.service.model.MSentryRole; @@ -1986,6 +1987,76 @@ public class SentryStore { } /** + * This returns a Mapping of Authz -> [Paths] + */ + public Map<String, Set<String>> retrieveFullPathsImage() { + Map<String, Set<String>> retVal = new HashMap<>(); + boolean rollbackTransaction = true; + PersistenceManager pm = null; + + try { + pm = openTransaction(); + Query query = pm.newQuery(MAuthzPathsMapping.class); + List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>) query.execute(); + for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) { + retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths()); + } + + rollbackTransaction = false; + commitTransaction(pm); + return retVal; + } finally { + if (rollbackTransaction) { + rollbackTransaction(pm); + } + } + } + + public CommitContext createAuthzPathsMapping(String hiveObj, + Set<String> paths) throws SentryNoSuchObjectException, SentryAlreadyExistsException { + + boolean rollbackTransaction = true; + PersistenceManager pm = null; + + try { + pm = openTransaction(); + createAuthzPathsMappingCore(pm, hiveObj, paths); + CommitContext commit = commitUpdateTransaction(pm); + rollbackTransaction = false; + return commit; + } finally { + if (rollbackTransaction) { + rollbackTransaction(pm); + } + } + } + + private void createAuthzPathsMappingCore(PersistenceManager pm, String authzObj, + Set<String> paths) throws SentryAlreadyExistsException { + + MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMapping(pm, authzObj); + + if (mAuthzPathsMapping == null) { + mAuthzPathsMapping = + new MAuthzPathsMapping(authzObj, paths, System.currentTimeMillis()); + pm.makePersistent(mAuthzPathsMapping); + } else { + throw new SentryAlreadyExistsException("AuthzObj: " + authzObj); + } + } + + /** + * Get the MAuthzPathsMapping object from authzObj + */ + public MAuthzPathsMapping getMAuthzPathsMapping(PersistenceManager pm, String authzObj) { + Query query = pm.newQuery(MAuthzPathsMapping.class); + query.setFilter("this.authzObjName == t"); + query.declareParameters("java.lang.String t"); + query.setUnique(true); + return (MAuthzPathsMapping) query.execute(authzObj); + } + + /** * This thread exists to clean up "orphaned" privilege rows in the database. * These rows aren't removed automatically due to the fact that there is * a many-to-many mapping between the roles and privileges, and the http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index bc7fe12..3ef1cf7 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -2079,6 +2079,18 @@ public class TestSentryStore extends org.junit.Assert { } } + @Test + public void testAuthzPathsMapping() throws Exception { + long seqId = sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1")).getSequenceId(); + long actualSeqId = sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2")).getSequenceId(); + assertEquals(seqId + 1, actualSeqId); + + Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage(); + assertEquals(2, pathsImage.size()); + assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1")); + } + + protected static void addGroupsToUser(String user, String... groupNames) { policyFile.addGroupsToUser(user, groupNames); }
