Repository: falcon Updated Branches: refs/heads/master ef57d29ad -> 4746e039a
FALCON-2147 Excess logging due to SlaMonitoring service and Backlog e⦠â¦mitter service Author: Pallavi Rao <[email protected]> Reviewers: @sandeepSamudrala Closes #286 from pallavi-rao/2147 and squashes the following commits: b3cc9d0 [Pallavi Rao] Comments addressed 7069e2f [Pallavi Rao] Removing excessive user logging in in BacklogMetricEmitter cacdc1d [Pallavi Rao] Addressed comment 4ddd4dd [Pallavi Rao] FALCON-2147 Excess logging due to SlaMonitoring service and Backlog emitter service Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4746e039 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4746e039 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4746e039 Branch: refs/heads/master Commit: 4746e039a1522af375a4e877848a6274b2de1c66 Parents: ef57d29 Author: Pallavi Rao <[email protected]> Authored: Thu Oct 20 13:43:24 2016 +0530 Committer: Pallavi Rao <[email protected]> Committed: Thu Oct 20 13:43:24 2016 +0530 ---------------------------------------------------------------------- .../falcon/workflow/WorkflowEngineFactory.java | 4 ++-- .../workflow/engine/OozieWorkflowEngine.java | 2 +- .../service/BacklogMetricEmitterService.java | 19 +++++++++++++------ .../falcon/service/EntitySLAAlertService.java | 5 ++--- .../service/EntitySLAMonitoringService.java | 19 +++++++++---------- 5 files changed, 27 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java b/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java index c713712..e5d0da2 100644 --- a/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java +++ b/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java @@ -56,7 +56,7 @@ public final class WorkflowEngineFactory { LOG.debug("Returning native workflow engine for entity {}", entity.getName()); return nativeWorkflowEngine; } - LOG.debug("Returning configured workflow engine for entity {}.", entity); + LOG.debug("Returning configured workflow engine for entity {}", (entity == null)? null : entity.getName()); return getWorkflowEngine(); } @@ -70,7 +70,7 @@ public final class WorkflowEngineFactory { throws FalconException { // If entity is null or not schedulable and the engine property is not specified, return the configured WE. if (entity == null || !entity.getEntityType().isSchedulable()) { - LOG.debug("Returning configured workflow engine for entity {}.", entity); + LOG.debug("Returning configured workflow engine for entity {}", (entity == null)? null : entity.getName()); return getWorkflowEngine(); } http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java index 394600c..6964200 100644 --- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java +++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java @@ -371,7 +371,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine { //Load bundle as coord info is not returned in getBundleJobsInfo() BundleJob bundle = getBundleInfo(clusterName, job.getId()); filteredJobs.add(bundle); - LOG.debug("Found bundle {} with app path {} and status {}", + LOG.trace("Found bundle {} with app path {} and status {}", job.getId(), job.getAppPath(), job.getStatus()); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java index 801ab36..d9ac386 100644 --- a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java +++ b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java @@ -17,6 +17,7 @@ */ package org.apache.falcon.service; +import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.LifeCycle; import org.apache.falcon.entity.EntityUtil; @@ -204,7 +205,7 @@ public final class BacklogMetricEmitterService implements FalconService, @Override public void run() { - LOG.debug("BacklogMetricEmitter running for entities"); + LOG.debug("Starting periodic check for backlog"); executor = new ScheduledThreadPoolExecutor(10); List<Future> futures = new ArrayList<>(); try { @@ -316,11 +317,7 @@ public final class BacklogMetricEmitterService implements FalconService, Date nominalTime; try { nominalTime = DATE_FORMAT.get().parse(nominalTimeStr); - if (entity.getACL().getOwner() != null && !entity.getACL().getOwner().isEmpty()) { - CurrentUser.authenticate(entity.getACL().getOwner()); - } else { - CurrentUser.authenticate(System.getProperty("user.name")); - } + authenticateUser(entity); if (wfEngine.isMissing(entity)) { LOG.info("Entity of name {} was deleted so removing instance of " + "nominaltime {} ", entity.getName(), nominalTimeStr); @@ -353,4 +350,14 @@ public final class BacklogMetricEmitterService implements FalconService, } } + private static void authenticateUser(Entity entity){ + if (!CurrentUser.isAuthenticated()) { + if (StringUtils.isNotBlank(entity.getACL().getOwner())) { + CurrentUser.authenticate(entity.getACL().getOwner()); + } else { + CurrentUser.authenticate(System.getProperty("user.name")); + } + } + } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java b/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java index 9b1a594..09c6695 100644 --- a/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java +++ b/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java @@ -76,6 +76,7 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList continue; } EntitySLAListener listener = ReflectionUtils.getInstanceByClassName(listenerClassName); + LOG.info("Registering listener {}" , listenerClassName); registerListener(listener); } } @@ -111,8 +112,7 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList if (pendingInstanceBeanList == null || pendingInstanceBeanList.isEmpty()){ return; } - - LOG.debug("In processSLACandidates :" + pendingInstanceBeanList.size()); + LOG.trace("In processSLACandidates :" + pendingInstanceBeanList.size()); try{ for (PendingInstanceBean pendingInstanceBean : pendingInstanceBeanList) { @@ -161,7 +161,6 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList @Override public void highSLAMissed(String entityName, String clusterName, EntityType entityType , Date nominalTime ) throws FalconException { - LOG.debug("Listners called..."); for (EntitySLAListener listener : listeners) { listener.highSLAMissed(entityName, clusterName, entityType, nominalTime); store.deleteEntityAlertInstance(entityName, clusterName, nominalTime, entityType.name()); http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java b/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java index 1e20a2b..816846d 100644 --- a/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java +++ b/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java @@ -303,14 +303,14 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList freq = StartupProperties.get().getProperty("entity.sla.lookAheadWindow.millis", "900000"); lookAheadWindowMillis = Integer.parseInt(freq); - LOG.debug("No old state exists at: {}, Initializing a clean state.", filePath.toString()); + LOG.info("Initializing EntitySLAMonitoringService from ", filePath.toString()); ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); executor.scheduleWithFixedDelay(new Monitor(), 0, statusCheckFrequencySeconds, TimeUnit.SECONDS); } public void makeFeedInstanceAvailable(String feedName, String clusterName, Date nominalTime) throws FalconException { - LOG.info("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName, + LOG.debug("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName, clusterName, nominalTime); List<Date> instances = (MONITORING_JDBC_STATE_STORE.getNominalInstances(feedName, clusterName, EntityType.FEED.toString())); @@ -369,7 +369,6 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList for(MonitoredEntityBean monitoredEntityBean : entityBeanList) { String entityName = monitoredEntityBean.getFeedName(); Entity entity = EntityUtil.getEntity(entityType, entityName); - LOG.debug("entityName:"+ entityName+"entity:"+entity); Set<String> clusters = EntityUtil.getClustersDefined(entity); List<org.apache.falcon.entity.v0.cluster.Cluster> cluster = new ArrayList(); for(String string : clusters){ @@ -391,7 +390,7 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList org.apache.falcon.entity.v0.cluster.Cluster currentCluster = EntityUtil.getEntity(EntityType.CLUSTER, entityCluster.getName()); nextInstanceTime = EntityUtil.getNextStartTime(entity, currentCluster, nextInstanceTime); - LOG.info("nextInstanceTime:"+ nextInstanceTime + "entityName:"+entityName); + LOG.trace("nextInstanceTime:"+ nextInstanceTime + "entityName:"+entityName); Date endDate; if (entityType.equals(EntityType.FEED.toString())){ endDate = FeedHelper.getClusterValidity((Feed) entity, currentCluster.getName()).getEnd(); @@ -400,7 +399,8 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList currentCluster.getName()).getEnd(); } while (nextInstanceTime.before(to) && nextInstanceTime.before(endDate)) { - LOG.info("Adding instance={} for <entity,cluster>={}", nextInstanceTime, key); + LOG.trace("Adding pending instance={} for <entity,cluster>={}; entityType={}", + nextInstanceTime, key, entityType); instances.add(nextInstanceTime); nextInstanceTime = new Date(nextInstanceTime.getTime() + ONE_MS); nextInstanceTime = EntityUtil.getNextStartTime(entity, currentCluster, nextInstanceTime); @@ -420,7 +420,6 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList * Checks the availability of all the pendingInstances and removes the ones which have become available. */ private void checkPendingInstanceAvailability(String entityType) throws FalconException { - LOG.debug("Size "+MONITORING_JDBC_STATE_STORE.getAllMonitoredEntity().size()); if (MONITORING_JDBC_STATE_STORE.getAllPendingInstances() == null){ LOG.info("Returning as size of pending instance is zero"); return; @@ -445,26 +444,26 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList authenticateUser(entity); try { if (entityType.equals(EntityType.PROCESS.toString())){ - LOG.debug("Checking instance availability status for entity:{}, cluster:{}, " + LOG.trace("Checking instance availability status for entity:{}, cluster:{}, " + "instanceTime:{}", entity.getName(), clusterName, nominalTime, entityType); AbstractWorkflowEngine wfEngine = WorkflowEngineFactory.getWorkflowEngine(); InstancesResult instancesResult = wfEngine.getStatus(entity, nominalTime, nominalTime, null, null); if (instancesResult.getStatus().equals(APIResult.Status.SUCCEEDED)){ - LOG.debug("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is available.", + LOG.trace("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is available.", entity.getName(), clusterName, nominalTime); return true; } return false; } if (entityType.equals(EntityType.FEED.toString())){ - LOG.debug("Checking instance availability status for feed:{}, cluster:{}, instanceTime:{}", + LOG.trace("Checking instance availability status for feed:{}, cluster:{}, instanceTime:{}", entity.getName(), clusterName, nominalTime); FeedInstanceStatus.AvailabilityStatus status = FeedHelper.getFeedInstanceStatus((Feed) entity, clusterName, nominalTime); if (status.equals(FeedInstanceStatus.AvailabilityStatus.AVAILABLE) || status.equals(FeedInstanceStatus.AvailabilityStatus.EMPTY)) { - LOG.debug("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.", entity.getName(), + LOG.trace("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.", entity.getName(), clusterName, nominalTime); return true; }
