Repository: incubator-ranger Updated Branches: refs/heads/master ccbc468cd -> c3d48e666
RANGER-1042: Reduce clutter in tag-sync debug log Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c3d48e66 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c3d48e66 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c3d48e66 Branch: refs/heads/master Commit: c3d48e6660cd3f9aaf4b202472503fda81709275 Parents: ccbc468 Author: Abhay Kulkarni <[email protected]> Authored: Fri Jun 17 15:11:11 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Jun 17 22:39:45 2016 -0700 ---------------------------------------------------------------------- .../ranger/tagsync/process/TagSynchronizer.java | 13 ++++- .../source/atlas/AtlasNotificationMapper.java | 58 +++++++++++++++----- .../tagsync/source/atlas/AtlasTagSource.java | 7 +-- 3 files changed, 59 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c3d48e66/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index ac4dcd0..b1b17cd 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -19,13 +19,18 @@ package org.apache.ranger.tagsync.process; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.model.TagSource; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.StringTokenizer; public class TagSynchronizer { @@ -261,6 +266,12 @@ public class TagSynchronizer { } } } + + if (CollectionUtils.isEmpty(initializedTagSourceNameList)) { + LOG.warn("TagSync is not configured for any tag-sources. No tags will be received by TagSync."); + LOG.warn("Please recheck configuration properties and tagsync environment to ensure that this is correct."); + } + if (LOG.isDebugEnabled()) { LOG.debug("<== TagSynchronizer.initializeTagSources(" + initializedTagSourceNameList + ")"); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c3d48e66/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index 510732b..793ac28 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -34,11 +34,46 @@ import org.apache.ranger.plugin.model.RangerTagDef; import org.apache.ranger.plugin.model.RangerTagDef.RangerTagAttributeDef; import org.apache.ranger.plugin.util.ServiceTags; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class AtlasNotificationMapper { private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class); + + private static Map<String, Long> unhandledEventTypes = new HashMap<String, Long>(); + + private static void logUnhandledEntityNotification(EntityNotification entityNotification) { + + final int REPORTING_INTERVAL_FOR_UNHANDLED_ENTITYTYPE_IN_MILLIS = 5 * 60 * 1000; // 5 minutes + + boolean loggingNeeded = false; + String entityTypeName = entityNotification != null && entityNotification.getEntity() != null ? + entityNotification.getEntity().getTypeName() : null; + + if (entityTypeName != null) { + Long timeInMillis = unhandledEventTypes.get(entityTypeName); + long currentTimeInMillis = System.currentTimeMillis(); + if (timeInMillis == null || + (currentTimeInMillis - timeInMillis) >= REPORTING_INTERVAL_FOR_UNHANDLED_ENTITYTYPE_IN_MILLIS) { + unhandledEventTypes.put(entityTypeName, currentTimeInMillis); + loggingNeeded = true; + } + } else { + LOG.error("EntityNotification contains NULL entity or NULL entity-type"); + } + + if (loggingNeeded) { + LOG.warn("Ignoring entity notification of type " + entityTypeName); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring entity notification of type " + entityTypeName); + } + } + @SuppressWarnings("unchecked") public static ServiceTags processEntityNotification(EntityNotification entityNotification) { @@ -48,23 +83,18 @@ public class AtlasNotificationMapper { try { IReferenceableInstance entity = entityNotification.getEntity(); - if (entity != null) { - if (AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) { - AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entityNotification.getEntity(), entityNotification.getAllTraits()); - if (entityNotification.getOperationType() == EntityNotification.OperationType.ENTITY_DELETE) { - // Special case for ENTITY_DELETE notifications - ret = buildServiceTagsForEntityDeleteNotification(entityWithTraits); - } else { - ret = buildServiceTags(entityWithTraits, null); - } + if (entity != null && AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) { + AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entityNotification.getEntity(), entityNotification.getAllTraits()); + if (entityNotification.getOperationType() == EntityNotification.OperationType.ENTITY_DELETE) { + // Special case for ENTITY_DELETE notifications + ret = buildServiceTagsForEntityDeleteNotification(entityWithTraits); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Ranger not interested in Entity Notification for entity-type " + entityNotification.getEntity().getTypeName()); - } + ret = buildServiceTags(entityWithTraits, null); } } else { - LOG.error("EntityNotification contains NULL entity"); + logUnhandledEntityNotification(entityNotification); } + } catch (Exception exception) { LOG.error("createServiceTags() failed!! ", exception); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c3d48e66/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java index fe54cd4..910a8e6 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java @@ -38,7 +38,8 @@ import org.apache.ranger.plugin.util.ServiceTags; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.Properties; +import java.util.List; public class AtlasTagSource extends AbstractTagSource { private static final Log LOG = LogFactory.getLog(AtlasTagSource.class); @@ -187,9 +188,7 @@ public class AtlasTagSource extends AbstractTagSource { } ServiceTags serviceTags = AtlasNotificationMapper.processEntityNotification(notification); - if (serviceTags == null) { - LOG.error("No ServiceTags built for notification :" + getPrintableEntityNotification(notification)); - } else { + if (serviceTags != null) { updateSink(serviceTags); } } else {
