Repository: incubator-sentry Updated Branches: refs/heads/master dd9edc666 -> feb8cbee6
SENTRY-664: Fix to ensure path updates are synced after namenode restart (Reviewed by Prasad) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/feb8cbee Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/feb8cbee Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/feb8cbee Branch: refs/heads/master Commit: feb8cbee60e2d8d36667737d527c602de3e15d13 Parents: dd9edc6 Author: Arun Suresh <Arun Suresh> Authored: Wed Mar 4 11:52:53 2015 -0800 Committer: Arun Suresh <Arun Suresh> Committed: Wed Mar 4 11:52:53 2015 -0800 ---------------------------------------------------------------------- .../sentry/hdfs/SentryAuthorizationInfo.java | 24 ++++++++++++++------ .../org/apache/sentry/hdfs/MetastorePlugin.java | 9 ++++---- .../tests/e2e/hdfs/TestHDFSIntegration.java | 12 ++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/feb8cbee/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java index 2415890..f9a1f65 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java @@ -139,14 +139,14 @@ public class SentryAuthorizationInfo implements Runnable { if ((newAuthzPaths != authzPaths)||(newAuthzPerms != authzPermissions)) { lock.writeLock().lock(); try { + LOG.warn("FULL Updated paths seq Num [old=" + + authzPaths.getLastUpdatedSeqNum() + "], [new=" + + newAuthzPaths.getLastUpdatedSeqNum() + "]"); authzPaths = newAuthzPaths; - if (LOG.isDebugEnabled()) { - LOG.debug("FULL Updated paths seq Num [" + authzPaths.getLastUpdatedSeqNum() + "]"); - } + LOG.warn("FULL Updated perms seq Num [old=" + + authzPermissions.getLastUpdatedSeqNum() + "], [new=" + + newAuthzPerms.getLastUpdatedSeqNum() + "]"); authzPermissions = newAuthzPerms; - if (LOG.isDebugEnabled()) { - LOG.debug("FULL Updated perms seq Num [" + authzPermissions.getLastUpdatedSeqNum() + "]"); - } } finally { lock.writeLock().unlock(); } @@ -162,12 +162,22 @@ public class SentryAuthorizationInfo implements Runnable { // one in the List.. all the remaining will be partial updates if (updates.size() > 0) { if (updates.get(0).hasFullImage()) { + LOG.warn("Process Update : FULL IMAGE " + + "[" + updateable.getClass() + "]" + + "[" + updates.get(0).getSeqNum() + "]"); updateable = (V)updateable.updateFull(updates.remove(0)); } // Any more elements ? if (!updates.isEmpty()) { + LOG.warn("Process Update : More updates.. " + + "[" + updateable.getClass() + "]" + + "[" + updateable.getLastUpdatedSeqNum() + "]" + + "[" + updates.size() + "]"); updateable.updatePartial(updates, lock); } + LOG.warn("Process Update : Finished updates.. " + + "[" + updateable.getClass() + "]" + + "[" + updateable.getLastUpdatedSeqNum() + "]"); } return updateable; } @@ -198,7 +208,7 @@ public class SentryAuthorizationInfo implements Runnable { } public void start() { - if (authzPaths != null) { + if ((authzPaths != null)||(authzPermissions != null)) { boolean success = false; try { success = update(); http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/feb8cbee/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java index f4964d6..5277eef 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java @@ -88,12 +88,11 @@ public class MetastorePlugin extends SentryMetastoreListenerPlugin { private UpdateableAuthzPaths authzPaths; private Lock notificiationLock; - //Initialized to some value > 1 so that the first update notification - // will trigger a full Image fetch - private final AtomicLong seqNum = new AtomicLong(5); + // Initialized to some value > 1. + private static final AtomicLong seqNum = new AtomicLong(5); - // For some reason, HMS sometimes restarts the plugin - private static volatile long lastSentSeqNum = -1; + // Has to match the value of seqNum + private static volatile long lastSentSeqNum = seqNum.get(); private volatile boolean syncSent = false; private final ExecutorService threadPool; http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/feb8cbee/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java index 4cb7275..955c68a 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java @@ -764,6 +764,18 @@ public class TestHDFSIntegration { verifyOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true); // END : Verify external table set location.. + // Restart HDFS to verify if things are fine after re-start.. + + // TODO : this is currently commented out since miniDFS.restartNameNode() does + // not work corectly on the version of hadoop sentry depends on + // This has been verified to work on a real cluster. + // Once miniDFS is fixed, this should be uncommented.. + // miniDFS.shutdown(); + // miniDFS.restartNameNode(true); + // miniDFS.waitActive(); + // verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); + // verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); + stmt.close(); conn.close(); }
