Repository: sentry Updated Branches: refs/heads/master 129f4ed9a -> 71283bda2
SENTRY-1951: Old SentryStore.retrieveFullPathsImage() should be removed (Arjun Mishra, reviewed by Alexander Kolbasov, Vadim Spector) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/71283bda Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/71283bda Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/71283bda Branch: refs/heads/master Commit: 71283bda28927e59ab7d85f3e081776c190db86a Parents: 129f4ed Author: Sergio Pena <[email protected]> Authored: Tue Dec 12 14:36:04 2017 -0600 Committer: Sergio Pena <[email protected]> Committed: Tue Dec 12 14:36:04 2017 -0600 ---------------------------------------------------------------------- .../apache/sentry/hdfs/TestImageRetriever.java | 5 +- .../hdfs/TestSentryHDFSServiceProcessor.java | 11 +- .../db/service/persistent/SentryStore.java | 61 ---- .../db/service/persistent/TestSentryStore.java | 284 ++++++++++++++++--- 4 files changed, 250 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java index b355630..d2d5391 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java @@ -20,7 +20,6 @@ package org.apache.sentry.hdfs; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; import org.apache.sentry.hdfs.service.thrift.TPathChanges; -import org.apache.sentry.provider.db.service.persistent.PathsImage; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.junit.Before; import org.junit.Ignore; @@ -54,8 +53,8 @@ public class TestImageRetriever { fullPathsImage.put("db1", Sets.newHashSet("/user/db1")); fullPathsImage.put("db1.table1", Sets.newHashSet("/user/db1/table1")); - Mockito.when(sentryStoreMock.retrieveFullPathsImage()) - .thenReturn(new PathsImage(fullPathsImage, 1, 1)); + Mockito.when(sentryStoreMock.retrieveFullPathsImageUpdate(root)) + .thenReturn(new PathsUpdate(1, 1, true)); imageRetriever = new PathImageRetriever(sentryStoreMock, root); pathsUpdate = imageRetriever.retrieveFullImage(); http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java index f09d1b2..578757e 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java @@ -25,7 +25,6 @@ import org.apache.sentry.hdfs.service.thrift.TAuthzUpdateResponse; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; import org.apache.sentry.provider.db.service.model.MSentryPathChange; import org.apache.sentry.provider.db.service.model.MSentryPermChange; -import org.apache.sentry.provider.db.service.persistent.PathsImage; import org.apache.sentry.provider.db.service.persistent.PermissionsImage; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.junit.BeforeClass; @@ -58,8 +57,9 @@ public class TestSentryHDFSServiceProcessor { public void testInitialHDFSSyncReturnsAFullImage() throws Exception { Mockito.when(sentryStoreMock.getLastProcessedImageID()) .thenReturn(1L); - Mockito.when(sentryStoreMock.retrieveFullPathsImage()) - .thenReturn(new PathsImage(new HashMap<String, Collection<String>>(), 1, 1)); + String[]prefixes = {"/"}; + Mockito.when(sentryStoreMock.retrieveFullPathsImageUpdate(prefixes)) + .thenReturn(new PathsUpdate(1, 1, true)); Mockito.when(sentryStoreMock.getLastProcessedPermChangeID()) .thenReturn(1L); @@ -84,8 +84,9 @@ public class TestSentryHDFSServiceProcessor { public void testRequestSyncUpdatesWhenNewImagesArePersistedReturnsANewFullImage() throws Exception { Mockito.when(sentryStoreMock.getLastProcessedImageID()) .thenReturn(2L); - Mockito.when(sentryStoreMock.retrieveFullPathsImage()) - .thenReturn(new PathsImage(new HashMap<String, Collection<String>>(), 3, 2)); + String[]prefixes = {"/"}; + Mockito.when(sentryStoreMock.retrieveFullPathsImageUpdate(prefixes)) + .thenReturn(new PathsUpdate(3, 2, true)); Mockito.when(sentryStoreMock.getLastProcessedPermChangeID()) .thenReturn(3L); http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index c730a03..6c4631f 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -2670,37 +2670,6 @@ public class SentryStore { } /** - * Retrieves an up-to-date hive paths snapshot. <p> - * It reads hiveObj to paths mapping from {@link MAuthzPathsMapping} table and - * gets the changeID of latest delta update, from {@link MSentryPathChange}, that - * the snapshot corresponds to. - * - * NOTE: this method used to be used in the actual code but is now only used for tests - * which should be refactored to use the new {@link #retrieveFullPathsImageUpdate} functionality. - * - * TODO: Remove retrieveFullPathsImage method and reimplement tests. - * - * @return an up-to-date hive paths snapshot contains mapping of hiveObj to < Paths >. - * For empty image return {@link #EMPTY_CHANGE_ID} and a empty map. - * @throws Exception - */ - public PathsImage retrieveFullPathsImage() throws Exception { - return (PathsImage) tm.executeTransaction( - new TransactionBlock() { - public Object execute(PersistenceManager pm) throws Exception { - // curChangeID could be 0 for the first full snapshot fetching - // from HMS. It does not have corresponding delta update. - pm.setDetachAllOnCommit(false); // No need to detach objects - long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPathChange.class); - long curImageID = getCurrentAuthzPathsSnapshotID(pm); - Map<String, Collection<String>> pathImage = retrieveFullPathsImageCore(pm, curImageID); - - return new PathsImage(pathImage, curChangeID, curImageID); - } - }); - } - - /** * Retrieves an up-to-date hive paths snapshot. * The image only contains PathsDump in it. * <p> @@ -2732,36 +2701,6 @@ public class SentryStore { } /** - * Retrieves an up-to-date hive paths snapshot from {@code MAuthzPathsMapping} table. - * The snapshot is represented by a snapshot ID, and a map from hiveObj to paths. - * - * @return a mapping of hiveObj to < Paths >. - */ - private Map<String, Collection<String>> retrieveFullPathsImageCore(PersistenceManager pm, - long currentSnapshotID) { - if (currentSnapshotID <= EMPTY_PATHS_SNAPSHOT_ID) { - return Collections.emptyMap(); - } - - Query query = pm.newQuery(MAuthzPathsMapping.class); - query.setFilter("this.authzSnapshotID == currentSnapshotID"); - query.declareParameters("long currentSnapshotID"); - Collection<MAuthzPathsMapping> authzToPathsMappings = - (Collection<MAuthzPathsMapping>) query.execute(currentSnapshotID); - - if (authzToPathsMappings.isEmpty()) { - return Collections.emptyMap(); - } - - Map<String, Collection<String>> retVal = new HashMap<>(authzToPathsMappings.size()); - for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) { - retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPathStrings()); - } - - return retVal; - } - - /** * Extract all paths and convert them into HMSPaths obect * @param pm Persistence manager * @param currentSnapshotID Image ID we are interested in http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index 24b11f7..b410027 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -2451,36 +2452,48 @@ public class TestSentryStore extends org.junit.Assert { @Test public void testPersistFullPathsImage() throws Exception { Map<String, Collection<String>> authzPaths = new HashMap<>(); + String[] prefixes = {"/user/hive/warehouse"}; // Makes sure that authorizable object could be associated // with different paths and can be properly persisted into database. authzPaths.put("db1.table1", Sets.newHashSet("/user/hive/warehouse/db2.db/table1.1", - "/user/hive/warehouse/db2.db/table1.2")); + "/user/hive/warehouse/db2.db/table1.2")); // Makes sure that the same MPaths object could be associated // with multiple authorizable and can be properly persisted into database. authzPaths.put("db1.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2.1", - "/user/hive/warehouse/db2.db/table2.2")); + "/user/hive/warehouse/db2.db/table2.2")); authzPaths.put("db2.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2.1", - "/user/hive/warehouse/db2.db/table2.3")); + "/user/hive/warehouse/db2.db/table2.3")); long notificationID = 11; sentryStore.persistFullPathsImage(authzPaths, notificationID); - PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); long savedNotificationID = sentryStore.getLastProcessedNotificationID(); - assertEquals(1, pathsImage.getCurImgNum()); - Map<String, Collection<String>> pathImage = pathsImage.getPathImage(); + assertEquals(1, pathsUpdate.getImgNum()); + TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap(); + + assertEquals(10, nodeMap.size()); + //First element is "/" + int rootId = pathDump.getRootId(); + TPathEntry root = nodeMap.get(rootId); + assertEquals("/", root.getPathElement()); + + Map<String, Collection<String>>pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, true); + assertEquals(3, pathImage.size()); for (Map.Entry<String, Collection<String>> entry : pathImage.entrySet()) { assertEquals(2, entry.getValue().size()); } assertEquals(2, pathImage.get("db2.table2").size()); assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/user/hive/warehouse/db2.db/table1.1", - "/user/hive/warehouse/db2.db/table1.2"), - pathImage.get("db1.table1"))); + "/user/hive/warehouse/db2.db/table1.2"), + pathImage.get("db1.table1"))); assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/user/hive/warehouse/db2.db/table2.1", - "/user/hive/warehouse/db2.db/table2.2"), - pathImage.get("db1.table2"))); + "/user/hive/warehouse/db2.db/table2.2"), + pathImage.get("db1.table2"))); assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/user/hive/warehouse/db2.db/table2.1", - "/user/hive/warehouse/db2.db/table2.3"), - pathImage.get("db2.table2"))); + "/user/hive/warehouse/db2.db/table2.3"), + pathImage.get("db2.table2"))); assertEquals(6, sentryStore.getMPaths().size()); assertEquals(notificationID, savedNotificationID); } @@ -2509,8 +2522,19 @@ public class TestSentryStore extends org.junit.Assert { long latestID = sentryStore.getLastProcessedNotificationID(); assertEquals(notificationID, latestID); + String []prefixes = {"/hive"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap(); + assertEquals(4, nodeMap.size()); + int rootId = pathDump.getRootId(); + TPathEntry root = nodeMap.get(rootId); + assertEquals("/", root.getPathElement()); + + Map<String, Collection<String>>pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, true); + // Check that retrieving a full paths image returns both paths updates - Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); assertEquals(2, pathsImage.size()); assertEquals(1, pathsImage.get("db1").size()); assertTrue(pathsImage.get("db1").contains("/hive/db1")); @@ -2530,6 +2554,28 @@ public class TestSentryStore extends org.junit.Assert { assertEquals("u2", pathsChanges.get(1).getNotificationHash()); } + private void buildPathsImageMap(Map<Integer, TPathEntry> nodeMap, TPathEntry node, String path, + Map<String, Collection<String>> pathsImage, boolean includeLeadingSlash) { + path = path.isEmpty()?node.getPathElement(): (path + node.getPathElement() + "/"); + TPathEntry tempNode = node; + for(int childVal:node.getChildren()) { + node = nodeMap.get(childVal); + buildPathsImageMap(nodeMap, node, path, pathsImage, includeLeadingSlash); + } + if(tempNode.getChildren().isEmpty() && node.getAuthzObjs() != null) { + for (String authz : node.getAuthzObjs()) { + if (!pathsImage.containsKey(authz)) { + pathsImage.put(authz, new LinkedList<>()); + } + if(includeLeadingSlash) { + pathsImage.get(authz).add(path.replaceAll("(/+$)", "")); + } else { + pathsImage.get(authz).add(path.replaceAll("(^/+)|(/+$)", "")); + } + } + } + } + @Test public void testPersistDuplicatedNotificationIdShouldBeAllowed() throws Exception { long notificationID = 1; @@ -2560,8 +2606,16 @@ public class TestSentryStore extends org.junit.Assert { sentryStore.addAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate); - PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); - Map<String, Collection<String>> pathImage = pathsImage.getPathImage(); + String[]prefixes = {"/"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals(4, nodeMap.size());//Tree size + assertEquals("/", root.getPathElement()); assertEquals(1, pathImage.size()); assertEquals(2, pathImage.get("db1.table").size()); assertEquals(2, sentryStore.getMPaths().size()); @@ -2578,7 +2632,15 @@ public class TestSentryStore extends org.junit.Assert { delUpdate.newPathChange("db1.table") .addToDelPaths(Arrays.asList("db1", "tbl1")); sentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate); - pathImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(3, nodeMap.size());//Tree size assertEquals(1, pathImage.size()); assertEquals(1, pathImage.get("db1.table").size()); assertEquals(1, sentryStore.getMPaths().size()); @@ -2595,7 +2657,15 @@ public class TestSentryStore extends org.junit.Assert { delAllupdate.newPathChange("db1.table") .addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS)); sentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate); - pathImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(1, nodeMap.size());//Tree size assertEquals(0, pathImage.size()); assertEquals(0, sentryStore.getMPaths().size()); @@ -2617,7 +2687,16 @@ public class TestSentryStore extends org.junit.Assert { "user/hive/warehouse/db1.db/table1/p1")); authzPaths.put("db1.table2", Sets.newHashSet("user/hive/warehouse/db1.db/table2")); sentryStore.persistFullPathsImage(authzPaths, lastNotificationId); - Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + String[]prefixes = {"/user/hive/warehouse"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(8, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); @@ -2629,7 +2708,15 @@ public class TestSentryStore extends org.junit.Assert { .addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); sentryStore.renameAuthzPathsMapping("db1.table1", "db1.newTable1", "user/hive/warehouse/db1.db/table1", "user/hive/warehouse/db1.db/newTable1", renameUpdate); - pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(9, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); assertEquals(3, sentryStore.getMPaths().size()); assertTrue(pathsImage.containsKey("db1.newTable1")); @@ -2650,7 +2737,15 @@ public class TestSentryStore extends org.junit.Assert { renameUpdate.newPathChange("db1.newTable2") .addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); sentryStore.renameAuthzObj("db1.newTable1", "db1.newTable2", renameUpdate); - pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(9, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); assertEquals(3, sentryStore.getMPaths().size()); assertTrue(pathsImage.containsKey("db1.newTable2")); @@ -2675,7 +2770,15 @@ public class TestSentryStore extends org.junit.Assert { "user/hive/warehouse/db1.db/newTable1", "user/hive/warehouse/db1.db/newTable2", update); - pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(9, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); assertEquals(3, sentryStore.getMPaths().size()); assertTrue(pathsImage.containsKey("db1.newTable2")); @@ -2694,7 +2797,6 @@ public class TestSentryStore extends org.junit.Assert { @Test public void testPersistAndReplaceANewPathsImage() throws Exception { Map<String, Collection<String>> authzPaths = new HashMap<>(); - PathsImage pathsImage; long notificationID = 1; // First image to persist (this will be replaced later) @@ -2703,8 +2805,17 @@ public class TestSentryStore extends org.junit.Assert { authzPaths.put("db1.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2.1", "/user/hive/warehouse/db2.db/table2.2")); sentryStore.persistFullPathsImage(authzPaths, notificationID); - pathsImage = sentryStore.retrieveFullPathsImage(); - assertEquals(1, pathsImage.getCurImgNum()); + String[]prefixes = {"/user/hive/warehouse"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, true); + + assertEquals("/", root.getPathElement()); + assertEquals(9, nodeMap.size());//Tree size + assertEquals(1, pathsUpdate.getImgNum()); // Second image to persist (it should replace first image) authzPaths.clear(); @@ -2716,9 +2827,17 @@ public class TestSentryStore extends org.junit.Assert { "/another-warehouse/db2.db/table2.3")); sentryStore.persistFullPathsImage(authzPaths, notificationID+1); - pathsImage = sentryStore.retrieveFullPathsImage(); - assertEquals(2, pathsImage.getCurImgNum()); - Map<String, Collection<String>> pathImage = pathsImage.getPathImage(); + prefixes = new String[]{"/another-warehouse"}; + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, true); + + assertEquals("/", root.getPathElement()); + assertEquals(8, nodeMap.size());//Tree size + assertEquals(2, pathsUpdate.getImgNum()); assertEquals(3, pathImage.size()); for (Map.Entry<String, Collection<String>> entry : pathImage.entrySet()) { @@ -2763,8 +2882,16 @@ public class TestSentryStore extends org.junit.Assert { newAddUpdate.newPathChange("db2.table").addToAddPaths(Arrays.asList("db2", "tbl1")); newAddUpdate.newPathChange("db2.table").addToAddPaths(Arrays.asList("db2", "tbl2")); sentryStore.addAuthzPathsMapping("db2.table", Sets.newHashSet("db2/tbl1", "db2/tbl2"), newAddUpdate); - PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); - Map<String, Collection<String>> pathImage = pathsImage.getPathImage(); + String[]prefixes = {"/"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(10, nodeMap.size());//Tree size assertEquals(2, pathImage.size()); assertEquals(2, pathImage.get("db2.table").size()); assertEquals(4, sentryStore.getMPaths().size()); @@ -2774,8 +2901,15 @@ public class TestSentryStore extends org.junit.Assert { UniquePathsUpdate delUpdate = new UniquePathsUpdate("u3", notificationID, false); delUpdate.newPathChange("db2.table").addToDelPaths(Arrays.asList("db2", "tbl1")); sentryStore.deleteAuthzPathsMapping("db2.table", Sets.newHashSet("db2/tbl1"), delUpdate); - pathsImage = sentryStore.retrieveFullPathsImage(); - pathImage = pathsImage.getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(9, nodeMap.size());//Tree size assertEquals(2, pathImage.size()); assertEquals(1, pathImage.get("db2.table").size()); assertEquals(3, sentryStore.getMPaths().size()); @@ -2815,12 +2949,21 @@ public class TestSentryStore extends org.junit.Assert { .addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); sentryStore.renameAuthzPathsMapping("db3.table1", "db1.newTable1", "/another-warehouse/db3.db/table1.1", "user/hive/warehouse/db1.db/newTable1", renameUpdate); - Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + String[]prefixes = {"/"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, true); + + assertEquals("/", root.getPathElement()); + assertEquals(11, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); assertEquals(4, sentryStore.getMPaths().size()); assertTrue(pathsImage.containsKey("db1.newTable1")); assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/another-warehouse/db3.db/table1.2", - "user/hive/warehouse/db1.db/newTable1"), + "/user/hive/warehouse/db1.db/newTable1"), pathsImage.get("db1.newTable1"))); // Update path of 'db1.newTable2' from 'db1.newTable1' to 'db1.newTable2' @@ -2834,7 +2977,15 @@ public class TestSentryStore extends org.junit.Assert { "user/hive/warehouse/db1.db/newTable1", "user/hive/warehouse/db1.db/newTable2", update); - pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(12, nodeMap.size());//Tree size assertEquals(3, pathsImage.size()); assertEquals(5, sentryStore.getMPaths().size()); assertTrue(pathsImage.containsKey("db1.newTable2")); @@ -3275,7 +3426,16 @@ public class TestSentryStore extends org.junit.Assert { "user/hive/warehouse/db1.db/table1/p1")); authzPaths.put("db1.table2", Sets.newHashSet("user/hive/warehouse/db1.db/table2")); sentryStore.persistFullPathsImage(authzPaths, lastNotificationId); - Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + String[]prefixes = {"/user/hive/warehouse"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(8, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); if (lastNotificationId == null) { @@ -3291,7 +3451,15 @@ public class TestSentryStore extends org.junit.Assert { .addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); sentryStore.renameAuthzPathsMapping("db1.table1", "db1.newTable1", "user/hive/warehouse/db1.db/table1", "user/hive/warehouse/db1.db/newTable1", renameUpdate); - pathsImage = sentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathsImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathsImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(9, nodeMap.size());//Tree size assertEquals(2, pathsImage.size()); assertEquals(3, sentryStore.getMPaths().size()); assertTrue(pathsImage.containsKey("db1.newTable1")); @@ -3333,8 +3501,16 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(sentryStore.isAuthzPathsMappingEmpty(), true); sentryStore.addAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate); - PathsImage pathsImage = sentryStore.retrieveFullPathsImage(); - Map<String, Collection<String>> pathImage = pathsImage.getPathImage(); + String[]prefixes = {"/"}; + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(4, nodeMap.size());//Tree size assertEquals(1, pathImage.size()); assertEquals(2, pathImage.get("db1.table").size()); assertEquals(2, sentryStore.getMPaths().size()); @@ -3364,8 +3540,16 @@ public class TestSentryStore extends org.junit.Assert { localSentryStore.addAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate); - PathsImage pathsImage = localSentryStore.retrieveFullPathsImage(); - Map<String, Collection<String>> pathImage = pathsImage.getPathImage(); + String[]prefixes = {"/"}; + PathsUpdate pathsUpdate = localSentryStore.retrieveFullPathsImageUpdate(prefixes); + TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump(); + Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap(); + TPathEntry root = nodeMap.get(pathsDump.getRootId()); + Map<String, Collection<String>> pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(4, nodeMap.size());//Tree size assertEquals(1, pathImage.size()); assertEquals(2, pathImage.get("db1.table").size()); assertEquals(2, localSentryStore.getMPaths().size()); @@ -3379,7 +3563,15 @@ public class TestSentryStore extends org.junit.Assert { delUpdate.newPathChange("db1.table") .addToDelPaths(Arrays.asList("db1", "tbl1")); localSentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate); - pathImage = localSentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = localSentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(3, nodeMap.size());//Tree size assertEquals(1, pathImage.size()); assertEquals(1, pathImage.get("db1.table").size()); assertEquals(1, localSentryStore.getMPaths().size()); @@ -3393,7 +3585,15 @@ public class TestSentryStore extends org.junit.Assert { delAllupdate.newPathChange("db1.table") .addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS)); localSentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate); - pathImage = localSentryStore.retrieveFullPathsImage().getPathImage(); + pathsUpdate = localSentryStore.retrieveFullPathsImageUpdate(prefixes); + pathsDump = pathsUpdate.toThrift().getPathsDump(); + nodeMap = pathsDump.getNodeMap(); + root = nodeMap.get(pathsDump.getRootId()); + pathImage = new HashMap<>(); + buildPathsImageMap(nodeMap, root, "", pathImage, false); + + assertEquals("/", root.getPathElement()); + assertEquals(1, nodeMap.size());//Tree size assertEquals(0, pathImage.size()); assertEquals(0, localSentryStore.getMPaths().size());
