SENTRY-1966: Improve logging of HMS sync data (paths and permissions) flowing from Sentry to NameNode (Vadim Spector, reviewed by Arjun Mishra)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/19efc0e4 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/19efc0e4 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/19efc0e4 Branch: refs/heads/akolb-cli Commit: 19efc0e44ad71e796c8fc8cac1cbd30b988f8ebb Parents: 186e755 Author: Vadim Spector <[email protected]> Authored: Tue Oct 3 12:29:05 2017 -0700 Committer: Vadim Spector <[email protected]> Committed: Tue Oct 3 12:45:31 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/sentry/hdfs/HMSPaths.java | 142 +++++++++++++++---- .../org/apache/sentry/hdfs/PathsUpdate.java | 6 + .../apache/sentry/hdfs/PermissionsUpdate.java | 6 + .../apache/sentry/hdfs/SentryAuthzUpdate.java | 27 ++++ .../sentry/hdfs/UpdateableAuthzPaths.java | 9 ++ .../sentry/hdfs/SentryAuthorizationInfo.java | 59 ++++++-- .../hdfs/SentryINodeAttributesProvider.java | 2 +- .../apache/sentry/hdfs/SentryPermissions.java | 25 ++++ .../sentry/hdfs/UpdateableAuthzPermissions.java | 9 ++ .../SentryHDFSServiceClientDefaultImpl.java | 23 +++ 10 files changed, 266 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java index 5e975cb..3919d60 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java @@ -70,6 +70,22 @@ public class HMSPaths implements AuthzPaths { return pathsElements; } + // used for more compact logging + static List<String> assemblePaths(List<List<String>> pathElements) { + if (pathElements == null) { + return Collections.emptyList(); + } + List<String> paths = new ArrayList<>(pathElements.size()); + for (List<String> path : pathElements) { + StringBuffer sb = new StringBuffer(); + for (String elem : path) { + sb.append(Path.SEPARATOR_CHAR).append(elem); + } + paths.add(sb.toString()); + } + return paths; + } + @VisibleForTesting enum EntryType { DIR(true), @@ -252,9 +268,10 @@ public class HMSPaths implements AuthzPaths { parent = null; } + @Override public String toString() { - return String.format("Entry[fullPath: %s, type: %s, authObject: %s]", - getFullPath(), type, authzObjsToString()); + return String.format("Entry[%s:%s -> authObj: %s]", + type, getFullPath(), authzObjsToString()); } private String authzObjsToString() { @@ -408,8 +425,8 @@ public class HMSPaths implements AuthzPaths { Entry prefix = findPrefixEntry(pathElements); if (prefix != null) { throw new IllegalArgumentException(String.format( - "Cannot add prefix '%s' under an existing prefix '%s'", - toPath(pathElements), prefix.getFullPath())); + "%s: createPrefix(%s): cannot add prefix under an existing prefix '%s'", + this, pathElements, prefix.getFullPath())); } return createChild(pathElements, EntryType.PREFIX, null); } @@ -421,7 +438,10 @@ public class HMSPaths implements AuthzPaths { // we only create the entry if is under a prefix, else we ignore it entry = createChild(pathElements, EntryType.AUTHZ_OBJECT, authzObj); } else { - LOG.debug("Skipping to create authzObjPath as it is outside of prefix. authObj={} pathElements={}", authzObj, pathElements); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s: createAuthzObjPath(%s, %s): outside of prefix, skipping", + this, authzObj, pathElements)); + } } return entry; } @@ -478,7 +498,7 @@ public class HMSPaths implements AuthzPaths { if (newParent.getChild(pathElem) != null) { LOG.warn(String.format( "Attempt to move %s to %s: entry with the same name %s already exists", - this.getFullPath(), newParent.getFullPath(), pathElem)); + this, newParent, pathElem)); return; } deleteFromParent(); @@ -642,6 +662,7 @@ public class HMSPaths implements AuthzPaths { private Map<String, Set<Entry>> authzObjToEntries; public HMSPaths() { + LOG.info(toString() + " (default) Initialized"); } public HMSPaths(String[] pathPrefixes) { @@ -663,6 +684,7 @@ public class HMSPaths implements AuthzPaths { } authzObjToEntries = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER); + LOG.info(toString() + " Initialized"); } void _addAuthzObject(String authzObj, List<String> authzObjPaths) { @@ -670,6 +692,10 @@ public class HMSPaths implements AuthzPaths { } void addAuthzObject(String authzObj, List<List<String>> authzObjPathElements) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s addAuthzObject(%s, %s)", + this, authzObj, assemblePaths(authzObjPathElements))); + } Set<Entry> previousEntries = authzObjToEntries.get(authzObj); Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size()); for (List<String> pathElements : authzObjPathElements) { @@ -677,7 +703,9 @@ public class HMSPaths implements AuthzPaths { if (e != null) { newEntries.add(e); } else { - LOG.warn("Ignoring path, no prefix"); + LOG.warn(String.format("%s addAuthzObject(%s, %s):" + + " Ignoring path %s, no prefix", + this, authzObj, assemblePaths(authzObjPathElements), pathElements)); } } authzObjToEntries.put(authzObj, newEntries); @@ -693,6 +721,10 @@ public class HMSPaths implements AuthzPaths { void addPathsToAuthzObject(String authzObj, List<List<String>> authzObjPathElements, boolean createNew) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s addPathsToAuthzObject(%s, %s, %b)", + this, authzObj, assemblePaths(authzObjPathElements), createNew)); + } Set<Entry> entries = authzObjToEntries.get(authzObj); if (entries != null) { Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size()); @@ -701,7 +733,11 @@ public class HMSPaths implements AuthzPaths { if (e != null) { newEntries.add(e); } else { - LOG.debug("Cannot create authz obj path for {} because it is outside of prefix", authzObj); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s addPathsToAuthzObject(%s, %s, %b):" + + " Cannot create authz obj for path %s because it is outside of prefix", + this, authzObj, assemblePaths(authzObjPathElements), createNew, pathElements)); + } } } entries.addAll(newEntries); @@ -709,8 +745,9 @@ public class HMSPaths implements AuthzPaths { if (createNew) { addAuthzObject(authzObj, authzObjPathElements); } else { - LOG.warn("Path was not added to AuthzObject, could not find key in authzObjToPath. authzObj = " + authzObj + - " authzObjPathElements=" + authzObjPathElements); + LOG.warn(String.format("%s addPathsToAuthzObject(%s, %s, %b):" + + " Path was not added to AuthzObject, could not find key in authzObjToPath", + this, authzObj, assemblePaths(authzObjPathElements), createNew)); } } } @@ -730,6 +767,10 @@ public class HMSPaths implements AuthzPaths { */ void deletePathsFromAuthzObject(String authzObj, List<List<String>> authzObjPathElements) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s deletePathsFromAuthzObject(%s, %s)", + this, authzObj, assemblePaths(authzObjPathElements))); + } Set<Entry> entries = authzObjToEntries.get(authzObj); if (entries != null) { Set<Entry> toDelEntries = new HashSet<Entry>(authzObjPathElements.size()); @@ -740,17 +781,24 @@ public class HMSPaths implements AuthzPaths { entry.deleteAuthzObject(authzObj); toDelEntries.add(entry); } else { - LOG.info("Path was not deleted from AuthzObject, path not registered. This is possible for implicit partition locations. authzObj = " + authzObj + " authzObjPathElements=" + authzObjPathElements); + LOG.warn(String.format("%s deletePathsFromAuthzObject(%s, %s):" + + " Path %s was not deleted from AuthzObject, path not registered." + + " This is possible for implicit partition locations", + this, authzObj, assemblePaths(authzObjPathElements), pathElements)); } } entries.removeAll(toDelEntries); } else { - LOG.info("Path was not deleted from AuthzObject, could not find key in authzObjToPath. authzObj = " + authzObj + - " authzObjPathElements=" + authzObjPathElements); + LOG.warn(String.format("%s deletePathsFromAuthzObject(%s, %s):" + + " Path was not deleted from AuthzObject, could not find key in authzObjToPath", + this, authzObj, assemblePaths(authzObjPathElements))); } } void deleteAuthzObject(String authzObj) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s deleteAuthzObject(%s)", this, authzObj)); + } Set<Entry> entries = authzObjToEntries.remove(authzObj); if (entries != null) { for (Entry entry : entries) { @@ -784,12 +832,21 @@ public class HMSPaths implements AuthzPaths { * @return Returns a set of authzObjects authzObject associated with this path. */ public Set<String> findAuthzObject(String[] pathElements, boolean isPartialOk) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s findAuthzObject(%s, %b)", + this, Arrays.toString(pathElements), isPartialOk)); + } // Handle '/' if (pathElements == null || pathElements.length == 0) { return null; } Entry entry = root.find(pathElements, isPartialOk); - return (entry != null) ? entry.getAuthzObjs() : null; + Set<String> authzObjSet = (entry != null) ? entry.getAuthzObjs() : null; + if ((authzObjSet == null || authzObjSet.isEmpty()) && LOG.isDebugEnabled()) { + LOG.debug(String.format("%s findAuthzObject(%s, %b) - no authzObject found", + this, Arrays.toString(pathElements), isPartialOk)); + } + return authzObjSet; } /* @@ -801,14 +858,16 @@ public class HMSPaths implements AuthzPaths { */ void renameAuthzObject(String oldName, List<List<String>> oldPathElems, String newName, List<List<String>> newPathElems) { - + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s})", + this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems))); + } if (oldPathElems == null || oldPathElems.isEmpty() || newPathElems == null || newPathElems.isEmpty() || newName == null || newName.equals(oldName)) { - LOG.warn(String.format( - "Unexpected state in renameAuthzObject, inputs invalid: " + - "oldName=%s newName=%s oldPath=%s newPath=%s", - oldName, newName, oldPathElems, newPathElems)); + LOG.warn(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s})" + + ": invalid inputs, skipping", + this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems))); return; } @@ -825,9 +884,9 @@ public class HMSPaths implements AuthzPaths { // Re-write authObj from oldName to newName. Set<Entry> entries = authzObjToEntries.get(oldName); if (entries == null) { - LOG.warn("Unexpected state in renameAuthzObject, cannot find oldName in authzObjToPath: " + - "oldName=" + oldName + " newName=" + newName + - " oldPath=" + oldPathElems + " newPath=" + newPathElems); + LOG.warn(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s}):" + + " cannot find oldName %s in authzObjToPath", + this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems), oldName)); } else { authzObjToEntries.put(newName, entries); for (Entry e : entries) { @@ -836,10 +895,10 @@ public class HMSPaths implements AuthzPaths { if (e.getAuthzObjs().contains(oldName)) { e.removeAuthzObj(oldName); } else { - LOG.warn("Unexpected state in renameAuthzObject, authzObjToPath has an " + - "entry <oldName,entries> where one of the entry does not have oldName : " + - "oldName=" + oldName + " newName=" + newName + - " oldPath=" + oldPathElems + " newPath=" + newPathElems); + LOG.warn(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s}):" + + " Unexpected state: authzObjToPath has an " + + "entry %s where one of the authz objects does not have oldName", + this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems), e)); } } } @@ -870,9 +929,40 @@ public class HMSPaths implements AuthzPaths { authzObjToEntries = mapping; } + /** + * For logging: collect all path entries into a list. + * + * Each Entry has informative toString() implementation, + * so we can print the returned value directly. + * + * Non-recursive traversal. + */ + public Collection<Entry> getAllEntries() { + Collection<Entry> entries = new ArrayList<>(); + Stack<Entry> stack = new Stack<>(); + stack.push(root); + while (!stack.isEmpty()) { + Entry entry = stack.pop(); + entries.add(entry); + for (Entry child : entry.childrenValues()) { // handles entry.children == null + stack.push(child); + } + } + return entries; + } + @Override public HMSPathsDumper getPathsDump() { return new HMSPathsDumper(this); } + @Override + public String toString() { + return String.format("%s:%s", getClass().getSimpleName(), Arrays.toString(prefixes)); + } + + public String dumpContent() { + return toString() + ": " + getAllEntries(); + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index 3996747..5ff2dc2 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -216,4 +216,10 @@ public class PathsUpdate implements Updateable.Update { return tPathsUpdate.equals(other.tPathsUpdate); } + @Override + public String toString() { + // TPathsUpdate implements toString() perfectly; null tPathsUpdate is ok + return getClass().getSimpleName() + "(" + tPathsUpdate + ")"; + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java index 392f9a6..0272396 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java @@ -155,4 +155,10 @@ public class PermissionsUpdate implements Updateable.Update { return tPermUpdate.equals(other.tPermUpdate); } + @Override + public String toString() { + // TPermissionsUpdate implements toString() perfectly; null tPermUpdate is ok + return getClass().getSimpleName() + "(" + tPermUpdate + ")"; + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java index 4cf439b..db11c1e 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java @@ -38,4 +38,31 @@ public class SentryAuthzUpdate { return pathUpdates; } + public String dumpContent() { + StringBuffer sb = new StringBuffer(getClass().getSimpleName()); + if (permUpdates != null && !permUpdates.isEmpty()) { + sb.append(", perms[").append(permUpdates.size()).append(']').append(permUpdates); + } + if (pathUpdates != null && !pathUpdates.isEmpty()) { + sb.append(", paths[").append(pathUpdates.size()).append(']').append(pathUpdates); + } + return sb.toString(); + } + + public boolean isEmpty() { + return (permUpdates == null || permUpdates.isEmpty()) && + (pathUpdates == null || pathUpdates.isEmpty()); + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(getClass().getSimpleName()); + if (permUpdates != null && !permUpdates.isEmpty()) { + sb.append(", perms[").append(permUpdates.size()).append(']'); + } + if (pathUpdates != null && !pathUpdates.isEmpty()) { + sb.append(", paths[").append(pathUpdates.size()).append(']'); + } + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java index cd97939..c967f4e 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java @@ -191,4 +191,13 @@ public class UpdateableAuthzPaths implements AuthzPaths, Updateable<PathsUpdate> public String getUpdateableTypeName() { return UPDATABLE_TYPE_NAME; } + + @Override + public String toString() { + return String.format("%s(%s, %s, %s)", getClass().getSimpleName(), seqNum, imgNum, paths); + } + + public String dumpContent() { + return String.format("%s(%s, %s) ", getClass().getSimpleName(), seqNum, imgNum) + paths.dumpContent(); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java index 2724a55..61b0d10 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java @@ -87,15 +87,13 @@ public class SentryAuthorizationInfo implements Runnable { SentryAuthorizationConstants.CACHE_REFRESH_RETRY_WAIT_KEY, SentryAuthorizationConstants.CACHE_REFRESH_RETRY_WAIT_DEFAULT); - if (LOG.isDebugEnabled()) { - LOG.debug( - "Sentry authorization will enforced in the following HDFS locations: [{}]", + LOG.info("Sentry authorization will enforced in the following HDFS locations: [{}]", StringUtils.arrayToString(newPathPrefixes)); - } + setPrefixPaths(newPathPrefixes); - LOG.debug("Refresh interval [{}]ms, retry wait [{}]", + LOG.info("Refresh interval [{}]ms, retry wait [{}]", refreshIntervalMillisec, retryWaitMillisec); - LOG.debug("stale threshold [{}]ms", staleThresholdMillisec); + LOG.info("stale threshold [{}]ms", staleThresholdMillisec); authzPaths = new UpdateableAuthzPaths(newPathPrefixes); authzPermissions = new UpdateableAuthzPermissions(); @@ -134,6 +132,10 @@ public class SentryAuthorizationInfo implements Runnable { SentryAuthzUpdate updates = updater.getUpdates(); // Updates can be null if Sentry Service is un-reachable if (updates != null) { + if (updates.isEmpty()) { + return true; // no updates is a norm, it's still success + } + UpdateableAuthzPaths newAuthzPaths = processUpdates( updates.getPathUpdates(), authzPaths); UpdateableAuthzPermissions newAuthzPerms = processUpdates( @@ -145,16 +147,41 @@ public class SentryAuthorizationInfo implements Runnable { if (newAuthzPaths != authzPaths || newAuthzPerms != authzPermissions) { lock.writeLock().lock(); try { - LOG.debug(String.format("FULL Updated paths seq Num [old=%d], [new=%d] img Num [old=%d], [new=%d]", - authzPaths.getLastUpdatedSeqNum(), newAuthzPaths.getLastUpdatedSeqNum(), - authzPaths.getLastUpdatedImgNum(), newAuthzPaths.getLastUpdatedImgNum())); - authzPaths = newAuthzPaths; - LOG.debug(String.format("FULL Updated perms seq Num [old=%d], [new=%d]", + if (LOG.isDebugEnabled()) { + LOG.debug(updates.dumpContent()); + } + if (newAuthzPaths != authzPaths) { + LOG.info(String.format("FULL Updated paths seq Num [old=%d], [new=%d]", + authzPaths.getLastUpdatedSeqNum(), newAuthzPaths.getLastUpdatedSeqNum())); + authzPaths = newAuthzPaths; + if (LOG.isTraceEnabled()) { + LOG.trace(authzPaths.dumpContent()); + } + } + if (newAuthzPerms != authzPermissions) { + LOG.info(String.format("FULL Updated perms seq Num [old=%d], [new=%d]", authzPermissions.getLastUpdatedSeqNum(), newAuthzPerms.getLastUpdatedSeqNum())); - authzPermissions = newAuthzPerms; + authzPermissions = newAuthzPerms; + if (LOG.isTraceEnabled()) { + LOG.trace(authzPermissions.dumpContent()); + } + } } finally { lock.writeLock().unlock(); } + } else { + if (LOG.isDebugEnabled()) { + lock.writeLock().lock(); + try { + LOG.debug(updates.dumpContent()); + if (LOG.isTraceEnabled()) { + LOG.trace(newAuthzPaths.dumpContent()); + LOG.trace(newAuthzPerms.dumpContent()); + } + } finally { + lock.writeLock().unlock(); + } + } } return true; } @@ -169,7 +196,7 @@ public class SentryAuthorizationInfo implements Runnable { if (!updates.isEmpty()) { if (updates.get(0).hasFullImage()) { LOG.debug(String.format("Process Update : FULL IMAGE [%s][%d][%d]", - newUpdateable.getClass().getName(), + newUpdateable.getClass().getSimpleName(), updates.get(0).getSeqNum(), updates.get(0).getImgNum())); newUpdateable = (V)newUpdateable.updateFull(updates.remove(0)); @@ -177,20 +204,21 @@ public class SentryAuthorizationInfo implements Runnable { // Any more elements ? if (!updates.isEmpty()) { LOG.debug(String.format("Process Update : More updates.. [%s][%d][%d][%d]", - newUpdateable.getClass().getName(), + newUpdateable.getClass().getSimpleName(), newUpdateable.getLastUpdatedSeqNum(), newUpdateable.getLastUpdatedImgNum(), updates.size())); newUpdateable.updatePartial(updates, lock); } LOG.debug(String.format("Process Update : Finished updates.. [%s][%d][%d]", - newUpdateable.getClass().getName(), + newUpdateable.getClass().getSimpleName(), newUpdateable.getLastUpdatedSeqNum(), newUpdateable.getLastUpdatedImgNum())); } return newUpdateable; } + @Override public void run() { boolean success = false; try { @@ -241,6 +269,7 @@ public class SentryAuthorizationInfo implements Runnable { public void stop() { if (authzPaths != null) { + LOG.info(getClass().getSimpleName() + ": Stopping"); executor.shutdownNow(); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java index 23b831d..cf96df4 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java @@ -323,7 +323,7 @@ public class SentryINodeAttributesProvider extends INodeAttributeProvider @Override public void stop() { - LOG.debug("Stopping"); + LOG.debug(getClass().getSimpleName() + ": Stopping"); authzInfo.stop(); } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java index c01ff68..3b3a82e 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java @@ -49,6 +49,10 @@ public class SentryPermissions implements AuthzPermissions { public String getAuthzObj() { return authzObj; } + @Override + public String toString() { + return "PrivilegeInfo(" + authzObj + " --> " + roleToPermission + ")"; + } } public static class RoleInfo { @@ -71,6 +75,10 @@ public class SentryPermissions implements AuthzPermissions { public Set<String> getAllGroups() { return groups; } + @Override + public String toString() { + return "RoleInfo(" + role + " --> " + groups + ")"; + } } // Comparison of authorizable object should be case insensitive. @@ -203,4 +211,21 @@ public class SentryPermissions implements AuthzPermissions { public void addRoleInfo(RoleInfo roleInfo) { roles.put(roleInfo.role, roleInfo); } + + public String dumpContent() { + return new StringBuffer(getClass().getSimpleName()) + .append(": Privileges: ").append(privileges) + .append(", Roles: ").append(roles) + .append(", AuthzObjChildren: ").append(authzObjChildren) + .toString(); + } + + @Override + public String toString() { + return new StringBuffer(getClass().getSimpleName()) + .append(": Privileges: ").append(privileges.size()) + .append(", Roles: ").append(roles.size()) + .append(", AuthzObjChildren: ").append(authzObjChildren.size()) + .toString(); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java index bf13ce7..1505513 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java @@ -250,4 +250,13 @@ public class UpdateableAuthzPermissions implements AuthzPermissions, Updateable< return UPDATABLE_TYPE_NAME; } + @Override + public String toString() { + return String.format("%s(%s, %s)", getClass().getSimpleName(), seqNum, perms); + } + + public String dumpContent() { + return String.format("%s(%s) ", getClass().getSimpleName(), seqNum) + perms.dumpContent(); + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java index 30d8adf..9115697 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java @@ -37,6 +37,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import static org.apache.sentry.hdfs.service.thrift.sentry_hdfs_serviceConstants.UNUSED_PATH_UPDATE_IMG_NUM; /** @@ -46,6 +49,9 @@ import static org.apache.sentry.hdfs.service.thrift.sentry_hdfs_serviceConstants */ public class SentryHDFSServiceClientDefaultImpl implements SentryHDFSServiceClient, SentryConnection { + + private static final Logger LOGGER = LoggerFactory.getLogger(SentryHDFSServiceClientDefaultImpl.class); + private final boolean useCompactTransport; private Client client; private final SentryTransportPool transportPool; @@ -97,6 +103,7 @@ public class SentryHDFSServiceClientDefaultImpl try { TAuthzUpdateRequest updateRequest = new TAuthzUpdateRequest(permSeqNum, pathSeqNum, pathImgNum); TAuthzUpdateResponse sentryUpdates = client.get_authz_updates(updateRequest); + List<PathsUpdate> pathsUpdates = Collections.emptyList(); if (sentryUpdates.getAuthzPathUpdate() != null) { pathsUpdates = new ArrayList<>(sentryUpdates.getAuthzPathUpdate().size()); @@ -113,6 +120,19 @@ public class SentryHDFSServiceClientDefaultImpl } } + if (LOGGER.isDebugEnabled() && !(permsUpdates.isEmpty() && pathsUpdates.isEmpty()) ) { + LOGGER.debug("getAllUpdatesFrom({},{},{}): permsUpdates[{}], pathsUpdates[{}]", + new Object[] { permSeqNum, pathSeqNum, pathImgNum, permsUpdates.size(), pathsUpdates.size() }); + if (LOGGER.isTraceEnabled()) { + if (!permsUpdates.isEmpty()) { + LOGGER.trace("permsUpdates{}", permsUpdates); + } + if (!pathsUpdates.isEmpty()) { + LOGGER.trace("pathsUpdates{}", pathsUpdates); + } + } + } + return new SentryAuthzUpdate(permsUpdates, pathsUpdates); } catch (Exception e) { throw new SentryHdfsServiceException("Thrift Exception occurred !!", e); @@ -136,6 +156,9 @@ public class SentryHDFSServiceClientDefaultImpl public void invalidate() { if (transport != null) { transportPool.invalidateTransport(transport); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("invalidate: " + transport); + } transport = null; } }
