Repository: sentry Updated Branches: refs/heads/master c302da781 -> 398a8a937
SENTRY-1931: NameNode only gets full snapshot once (Sergio Pena, Reviewed by: Alexander Kolbasov, Vamsee Yarlagadda) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/398a8a93 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/398a8a93 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/398a8a93 Branch: refs/heads/master Commit: 398a8a9378cb43fea7f66153df917b9eb3abd40c Parents: c302da7 Author: Vamsee Yarlagadda <[email protected]> Authored: Thu Sep 7 18:15:57 2017 -0700 Committer: Vamsee Yarlagadda <[email protected]> Committed: Thu Sep 7 18:15:57 2017 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java | 6 ++++-- .../src/main/java/org/apache/sentry/hdfs/PathsUpdate.java | 5 +++++ .../main/java/org/apache/sentry/hdfs/PermissionsUpdate.java | 5 +++++ .../src/main/java/org/apache/sentry/hdfs/Updateable.java | 2 ++ .../main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java | 6 +++++- .../main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java | 2 +- .../main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java | 3 ++- .../main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java | 2 +- .../java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java | 2 +- .../test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java | 6 ++++-- 10 files changed, 30 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java index 4503950..3b7fcef 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java @@ -29,7 +29,7 @@ import static org.apache.sentry.hdfs.Updateable.Update; * Sentry permissions are represented as {@link PermissionsUpdate} and HMS Paths * are represented as {@link PathsUpdate}. The delta update contains change * from a state to another. - * The {@link #retrieveDelta(long)} method obtains such delta update from a persistent storage. + * The {@link #retrieveDelta(long,long)} method obtains such delta update from a persistent storage. * Delta update is propagated to a consumer of Sentry, such as HDFS NameNode whenever * the consumer needs to synchronize the update. */ @@ -38,13 +38,15 @@ public interface DeltaRetriever<K extends Update> { /** * Retrieves all delta updates of type {@link Update} newer than or equal to * the given sequence number/change ID (inclusive) from a persistent storage. + * An image number may be used by the implementation to request deltas for a specific image. * An empty collection can be returned. * * @param seqNum the given seq number + * @param imgNum the given img number * @return delta updates of type K * @throws Exception when there is an error in operation on persistent storage */ - List<K> retrieveDelta(long seqNum) throws Exception; + List<K> retrieveDelta(long seqNum, long imgNum) throws Exception; /** * Checks if there the delta update is available, given the sequence number/change http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index ccc10ef..3996747 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -100,6 +100,11 @@ public class PathsUpdate implements Updateable.Update { return tPathsUpdate.getImgNum(); } + @Override + public void setImgNum(long imgNum) { + tPathsUpdate.setImgNum(imgNum); + } + public TPathsUpdate toThrift() { return tPathsUpdate; } http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java index 7aa60a3..392f9a6 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java @@ -68,6 +68,11 @@ public class PermissionsUpdate implements Updateable.Update { } @Override + public void setImgNum(long imgNum) { + throw new UnsupportedOperationException("setImgNum not used"); + } + + @Override public boolean hasFullImage() { return tPermUpdate.isHasfullImage(); } http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java index 795aac7..bbf3f13 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java @@ -40,6 +40,8 @@ public interface Updateable<K extends Updateable.Update> { long getImgNum(); + void setImgNum(long imgNum); + byte[] serialize() throws IOException; void deserialize(byte data[]) throws IOException; http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java index 819b6b2..cd97939 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java @@ -84,7 +84,11 @@ public class UpdateableAuthzPaths implements AuthzPaths, Updateable<PathsUpdate> lock.writeLock().lock(); } seqNum.set(update.getSeqNum()); - imgNum.set(update.getImgNum()); + + // Update the image ID only if the update has a new one + if (imgNum.get() < update.getImgNum()) { + imgNum.set(update.getImgNum()); + } LOG.debug("##### Updated paths seq Num [{}] img Num [{}]", seqNum.get(), imgNum.get()); } } finally { http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java index 5d74421..bb85c13 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java @@ -103,7 +103,7 @@ class DBUpdateForwarder<K extends Updateable.Update> { // Checks if newer deltas exist in the persistent storage. // If there are, return the list of delta updates. if (seqNum > SEQUENCE_NUMBER_UPDATE_UNINITIALIZED && deltaRetriever.isDeltaAvailable(seqNum)) { - List<K> deltas = deltaRetriever.retrieveDelta(seqNum); + List<K> deltas = deltaRetriever.retrieveDelta(seqNum, imgNum); if (!deltas.isEmpty()) { LOGGER.info("Newer delta updates are found up to sequence number: {}", curSeqNum); return deltas; http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java index fda7455..b2e45f9 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java @@ -44,7 +44,7 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate> { } @Override - public List<PathsUpdate> retrieveDelta(long seqNum) throws Exception { + public List<PathsUpdate> retrieveDelta(long seqNum, long imgNum) throws Exception { try (final Context timerContext = SentryHdfsMetricsUtil.getDeltaPathChangesTimer.time()) { List<MSentryPathChange> mSentryPathChanges = @@ -66,6 +66,7 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate> { PathsUpdate pathsUpdate = new PathsUpdate(); pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange()); pathsUpdate.setSeqNum(changeID); + pathsUpdate.setImgNum(imgNum); updates.add(pathsUpdate); } return updates; http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java index df6a0b0..6974d37 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java @@ -44,7 +44,7 @@ public class PermDeltaRetriever implements DeltaRetriever<PermissionsUpdate> { } @Override - public List<PermissionsUpdate> retrieveDelta(long seqNum) throws Exception { + public List<PermissionsUpdate> retrieveDelta(long seqNum, long imgNum) throws Exception { try (final Context timerContext = SentryHdfsMetricsUtil.getDeltaPermChangesTimer.time()) { Collection<MSentryPermChange> mSentryPermChanges = http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java index 8fbc100..830d00e 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java @@ -117,7 +117,7 @@ public class TestDBUpdateForwarder { Mockito.when(imageRetriever.getLatestImageID()).thenReturn(1L); Mockito.when(deltaRetriever.getLatestDeltaID()).thenReturn(3L); Mockito.when(deltaRetriever.isDeltaAvailable(2L)).thenReturn(true); - Mockito.when(deltaRetriever.retrieveDelta(2L)) + Mockito.when(deltaRetriever.retrieveDelta(2L, 1L)) .thenReturn(Arrays.asList(new PathsUpdate(3, 1, false))); List<PathsUpdate> updates = updater.getAllUpdatesFrom(2, 1); http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java index 63b8caf..60696cc 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java @@ -44,7 +44,7 @@ public class TestDeltaRetriever { .thenReturn(Collections.<MSentryPathChange>emptyList()); PathDeltaRetriever deltaRetriever = new PathDeltaRetriever(sentryStoreMock); - List<PathsUpdate> pathsUpdates = deltaRetriever.retrieveDelta(1); + List<PathsUpdate> pathsUpdates = deltaRetriever.retrieveDelta(1, 1); assertTrue(pathsUpdates.isEmpty()); } @@ -63,12 +63,14 @@ public class TestDeltaRetriever { .thenReturn(deltaPathChanges); deltaRetriever = new PathDeltaRetriever(sentryStoreMock); - pathsUpdates = deltaRetriever.retrieveDelta(1); + pathsUpdates = deltaRetriever.retrieveDelta(1, 3); assertEquals(2, pathsUpdates.size()); assertEquals(1, pathsUpdates.get(0).getSeqNum()); assertEquals(true, pathsUpdates.get(0).hasFullImage()); + assertEquals(3, pathsUpdates.get(0).getImgNum()); assertEquals(2, pathsUpdates.get(1).getSeqNum()); assertEquals(false, pathsUpdates.get(1).hasFullImage()); + assertEquals(3, pathsUpdates.get(1).getImgNum()); } }
