Repository: sentry Updated Branches: refs/heads/master 213a37092 -> d4172ba9f
SENTRY-1938: Sentry logs to provide more relevant information (Arjun Mishra, Reviewed by: Vamsee Yarlagadda, Alexander Kolbasov) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/d4172ba9 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/d4172ba9 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/d4172ba9 Branch: refs/heads/master Commit: d4172ba9f957610637978da5dfa4032464d4aafc Parents: 213a370 Author: Vamsee Yarlagadda <[email protected]> Authored: Wed Sep 13 14:40:20 2017 -0700 Committer: Vamsee Yarlagadda <[email protected]> Committed: Wed Sep 13 16:33:49 2017 -0700 ---------------------------------------------------------------------- .../apache/sentry/hdfs/DBUpdateForwarder.java | 34 ++++++++++++++------ .../org/apache/sentry/hdfs/SentryPlugin.java | 12 ++++--- .../service/thrift/NotificationProcessor.java | 2 +- 3 files changed, 32 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/d4172ba9/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java index 8a34d56..eae7861 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java @@ -43,10 +43,14 @@ class DBUpdateForwarder<K extends Updateable.Update> { private final DeltaRetriever<K> deltaRetriever; private static final Logger LOGGER = LoggerFactory.getLogger(DBUpdateForwarder.class); + //For logging purposes + private String retrieverType; + DBUpdateForwarder(final ImageRetriever<K> imageRetriever, - final DeltaRetriever<K> deltaRetriever) { + final DeltaRetriever<K> deltaRetriever) { this.imageRetriever = imageRetriever; this.deltaRetriever = deltaRetriever; + this.retrieverType = imageRetriever.getClass().getName(); } /** @@ -71,12 +75,12 @@ class DBUpdateForwarder<K extends Updateable.Update> { * e.g. {@link PathsUpdate} or {@link PermissionsUpdate} */ List<K> getAllUpdatesFrom(long seqNum, long imgNum) throws Exception { - LOGGER.debug("GetAllUpdatesFrom sequence number {}, image number {}", seqNum, imgNum); + LOGGER.debug(String.format("(%s) GetAllUpdatesFrom sequence number %d, image number %d", retrieverType, seqNum, imgNum)); // An imgNum >= 0 are valid values for image identifiers (0 means a full update is requested) if (imgNum >= IMAGE_NUMBER_UPDATE_UNINITIALIZED) { long curImgNum = imageRetriever.getLatestImageID(); - LOGGER.debug("Current image number is {}", curImgNum); + LOGGER.debug("({}) Current image number for is {}", retrieverType, curImgNum); if (curImgNum == IMAGE_NUMBER_UPDATE_UNINITIALIZED) { // Sentry has not fetched a full HMS snapshot yet. @@ -85,8 +89,13 @@ class DBUpdateForwarder<K extends Updateable.Update> { if (curImgNum > imgNum) { // In case a new HMS snapshot has been processed, then return a full paths image. - LOGGER.info("A newer full update is found with image number: {}", curImgNum); - return retrieveFullImage(); + List<K>fullImage = retrieveFullImage(); + //Only log if we have received full image + if( !fullImage.isEmpty()) { + LOGGER.info("({}) A newer full update with image number {} was found and being sent to HDFS", + retrieverType, curImgNum); + } + return fullImage; } } @@ -95,7 +104,7 @@ class DBUpdateForwarder<K extends Updateable.Update> { */ long curSeqNum = deltaRetriever.getLatestDeltaID(); - LOGGER.debug("Current sequence number is {}", curSeqNum); + LOGGER.debug("({}) Current sequence number is {}", retrieverType, curSeqNum); if (seqNum > curSeqNum) { // No new notifications were processed. @@ -107,20 +116,25 @@ class DBUpdateForwarder<K extends Updateable.Update> { if (seqNum > SEQUENCE_NUMBER_FULL_UPDATE_REQUEST && deltaRetriever.isDeltaAvailable(seqNum)) { List<K> deltas = deltaRetriever.retrieveDelta(seqNum, imgNum); if (!deltas.isEmpty()) { - LOGGER.info("Newer delta updates are found up to sequence number: {}", curSeqNum); + LOGGER.info("({}) Newer delta updates are found up to sequence number {} and being sent to HDFS", retrieverType, curSeqNum); return deltas; } } // If the sequence number is < 0 or the requested delta is not available, then we // return a full update. - LOGGER.info("A full update is returned due to an unavailable sequence number: {}", seqNum); - return retrieveFullImage(); + List<K>fullImage = retrieveFullImage(); + //Only log if we have received full image + if( fullImage != null && !fullImage.isEmpty()) { + LOGGER.info("({}) A full update is returned due to an unavailable sequence number: {}", + retrieverType, seqNum); + } + return fullImage; } private List<K> retrieveFullImage() throws Exception { if (SentryStateBank.isEnabled(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING)){ - LOGGER.debug("A full update is being loaded. Delaying updating client with full image until its finished."); + LOGGER.debug("({}) A full update is being loaded. Delaying updating client with full image until its finished.", retrieverType); return Collections.emptyList(); } else { http://git-wip-us.apache.org/repos/asf/sentry/blob/d4172ba9/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index 1318082..ee528be 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -140,6 +140,7 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen public List<PathsUpdate> getAllPathsUpdatesFrom(long pathSeqNum, long pathImgNum) throws Exception { if (!fullUpdateNN.get()) { // Most common case - Sentry is NOT handling a full update. + LOGGER.debug("Sending partial PATH update to NameNode for pathSeqNum {} and pathImgNum {}", pathSeqNum, pathImgNum); return pathsUpdater.getAllUpdatesFrom(pathSeqNum, pathImgNum); } @@ -147,7 +148,7 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen * Sentry is in the middle of signal-triggered full update. * It already got a full update from HMS */ - LOGGER.info("SIGNAL HANDLING: sending full update to NameNode"); + LOGGER.info("SIGNAL HANDLING: sending full PATH update to NameNode"); fullUpdateNN.set(false); // don't do full NN update till the next signal List<PathsUpdate> updates = pathsUpdater.getAllUpdatesFrom(SEQUENCE_NUMBER_UPDATE_UNINITIALIZED, IMAGE_NUMBER_UPDATE_UNINITIALIZED); @@ -166,20 +167,21 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen if (updates != null) { if (!updates.isEmpty()) { if (updates.get(0).hasFullImage()) { - LOGGER.info("SIGNAL HANDLING: Confirmed full update to NameNode"); + LOGGER.info("SIGNAL HANDLING: Confirmed full PATH update to NameNode for pathSeqNum {} and pathImgNum {}", pathSeqNum, pathImgNum); } else { - LOGGER.warn("SIGNAL HANDLING: Sending partial instead of full update to NameNode (???)"); + LOGGER.warn("SIGNAL HANDLING: Sending partial instead of full PATH update to NameNode for pathSeqNum {} and pathImgNum {} (???)", pathSeqNum, pathImgNum); } } else { - LOGGER.warn("SIGNAL HANDLING: Sending empty instead of full update to NameNode (???)"); + LOGGER.warn("SIGNAL HANDLING: Sending empty instead of full PATH update to NameNode for pathSeqNum {} and pathImgNum {} (???)", pathSeqNum, pathImgNum); } } else { - LOGGER.warn("SIGNAL HANDLING: returned NULL instead of full update to NameNode (???)"); + LOGGER.warn("SIGNAL HANDLING: returned NULL instead of full PATH update to NameNode for pathSeqNum {} and pathImgNum {} (???)", pathSeqNum, pathImgNum); } return updates; } public List<PermissionsUpdate> getAllPermsUpdatesFrom(long permSeqNum) throws Exception { + LOGGER.debug("Sending partial PERM update to NameNode for permSeqNum {}", permSeqNum); return permsUpdater.getAllUpdatesFrom(permSeqNum, UNUSED_PATH_UPDATE_IMG_NUM); } http://git-wip-us.apache.org/repos/asf/sentry/blob/d4172ba9/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java index 713fc3d..07a7db4 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java @@ -520,7 +520,7 @@ final class NotificationProcessor { } if (oldLocation.equals(newLocation)) { - LOGGER.info(String.format("Alter partition notification ignored as" + LOGGER.debug(String.format("Alter partition notification ignored as" + "location has not changed: AuthzObj = %s, Location = %s", dbName + "." + "." + tableName, oldLocation)); return false;
