Repository: hadoop Updated Branches: refs/heads/HDFS-7285 b57c9a35f -> 6b6a63bbb
YARN-3966. Fix excessive loggings in CapacityScheduler. (Jian He via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bc42d76 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bc42d76 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bc42d76 Branch: refs/heads/HDFS-7285 Commit: 4bc42d76e7fa53cb268cab0f9fe1fd8d8dbb17fd Parents: b6265d3 Author: Wangda Tan <wan...@apache.org> Authored: Fri Aug 7 09:46:57 2015 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Fri Aug 7 09:46:57 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 ++ .../scheduler/AbstractYarnScheduler.java | 10 +++--- .../scheduler/capacity/CapacityScheduler.java | 36 ++++++-------------- .../scheduler/capacity/LeafQueue.java | 18 +++++----- .../scheduler/capacity/ParentQueue.java | 14 ++++---- .../scheduler/common/fica/FiCaSchedulerApp.java | 8 +---- 6 files changed, 33 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc42d76/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fa04337..7d34eeb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -382,6 +382,8 @@ Release 2.8.0 - UNRELEASED YARN-3974. Refactor the reservation system test cases to use parameterized base test. (subru via curino) + YARN-3966. Fix excessive loggings in CapacityScheduler. (Jian He via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc42d76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 094f77d..d69600a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -215,12 +215,12 @@ public abstract class AbstractYarnScheduler protected synchronized void containerLaunchedOnNode( ContainerId containerId, SchedulerNode node) { // Get the application for the finished container - SchedulerApplicationAttempt application = getCurrentAttemptForContainer - (containerId); + SchedulerApplicationAttempt application = + getCurrentAttemptForContainer(containerId); if (application == null) { - LOG.info("Unknown application " - + containerId.getApplicationAttemptId().getApplicationId() - + " launched container " + containerId + " on node: " + node); + LOG.info("Unknown application " + containerId.getApplicationAttemptId() + .getApplicationId() + " launched container " + containerId + + " on node: " + node); this.rmContext.getDispatcher().getEventHandler() .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId)); return; http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc42d76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 0b39d35..1d353a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -795,8 +795,8 @@ public class CapacityScheduler extends application.getUser(), queue, queue.getActiveUsersManager(), rmContext, application.getPriority()); if (transferStateFromPreviousAttempt) { - attempt.transferStateFromPreviousAttempt(application - .getCurrentAppAttempt()); + attempt.transferStateFromPreviousAttempt( + application.getCurrentAppAttempt()); } application.setCurrentAppAttempt(attempt); @@ -899,8 +899,6 @@ public class CapacityScheduler extends FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId); if (application == null) { - LOG.info("Calling allocate on removed " + - "or non existant application " + applicationAttemptId); return EMPTY_ALLOCATION; } @@ -921,33 +919,26 @@ public class CapacityScheduler extends // make sure we aren't stopping/removing the application // when the allocate comes in if (application.isStopped()) { - LOG.info("Calling allocate on a stopped " + - "application " + applicationAttemptId); return EMPTY_ALLOCATION; } if (!ask.isEmpty()) { if(LOG.isDebugEnabled()) { - LOG.debug("allocate: pre-update" + - " applicationAttemptId=" + applicationAttemptId + - " application=" + application); + LOG.debug("allocate: pre-update " + applicationAttemptId + + " ask size =" + ask.size()); + application.showRequests(); } - application.showRequests(); - + // Update application requests if (application.updateResourceRequests(ask)) { updateDemandForQueue = (LeafQueue) application.getQueue(); } - LOG.debug("allocate: post-update"); - application.showRequests(); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("allocate:" + - " applicationAttemptId=" + applicationAttemptId + - " #ask=" + ask.size()); + if(LOG.isDebugEnabled()) { + LOG.debug("allocate: post-update"); + application.showRequests(); + } } application.updateBlacklist(blacklistAdditions, blacklistRemovals); @@ -1018,7 +1009,6 @@ public class CapacityScheduler extends for (ContainerStatus completedContainer : completedContainers) { ContainerId containerId = completedContainer.getContainerId(); RMContainer container = getRMContainer(containerId); - LOG.debug("Container FINISHED: " + containerId); completedContainer(container, completedContainer, RMContainerEventType.FINISHED); if (container != null) { @@ -1481,9 +1471,6 @@ public class CapacityScheduler extends queue.completedContainer(clusterResource, application, node, rmContainer, containerStatus, event, null, true); - LOG.info("Application attempt " + application.getApplicationAttemptId() - + " released container " + container.getId() + " on node: " + node - + " with event: " + event); if (containerStatus.getExitStatus() == ContainerExitStatus.PREEMPTED) { schedulerHealth.updatePreemption(Time.now(), container.getNodeId(), container.getId(), queue.getQueuePath()); @@ -1783,8 +1770,7 @@ public class CapacityScheduler extends .equals(DefaultResourceCalculator.class.getName())) { return EnumSet.of(SchedulerResourceTypes.MEMORY); } - return EnumSet - .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU); + return EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc42d76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a71cc68..2691c33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1125,12 +1125,8 @@ public class LeafQueue extends AbstractCSQueue { // Inform the ordering policy orderingPolicy.containerReleased(application, rmContainer); - releaseResource(clusterResource, application, - container.getResource(), node.getPartition(), rmContainer); - LOG.info("completedContainer" + - " container=" + container + - " queue=" + this + - " cluster=" + clusterResource); + releaseResource(clusterResource, application, container.getResource(), + node.getPartition(), rmContainer); } } @@ -1203,10 +1199,12 @@ public class LeafQueue extends AbstractCSQueue { User user = getUser(userName); user.releaseContainer(resource, nodePartition); metrics.setAvailableResourcesToUser(userName, application.getHeadroom()); - - LOG.info(getQueueName() + - " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + - " user=" + userName + " user-resources=" + user.getUsed()); + + if (LOG.isDebugEnabled()) { + LOG.debug(getQueueName() + + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + + " user=" + userName + " user-resources=" + user.getUsed()); + } } private void updateAbsoluteCapacityResource(Resource clusterResource) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc42d76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 725aea1..e01204c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -629,12 +629,9 @@ public class ParentQueue extends AbstractCSQueue { super.releaseResource(clusterResource, rmContainer.getContainer() .getResource(), node.getPartition()); - LOG.info("completedContainer" + - " queue=" + getQueueName() + - " usedCapacity=" + getUsedCapacity() + - " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + - " used=" + queueUsage.getUsed() + - " cluster=" + clusterResource); + if (LOG.isDebugEnabled()) { + LOG.debug("completedContainer " + this + ", cluster=" + clusterResource); + } // Note that this is using an iterator on the childQueues so this can't // be called if already within an iterator for the childQueues. Like @@ -646,8 +643,9 @@ public class ParentQueue extends AbstractCSQueue { CSQueue csqueue = iter.next(); if(csqueue.equals(completedChildQueue)) { iter.remove(); - LOG.info("Re-sorting completed queue: " + csqueue.getQueuePath() + - " stats: " + csqueue); + if (LOG.isDebugEnabled()) { + LOG.debug("Re-sorting completed queue: " + csqueue); + } childQueues.add(csqueue); break; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc42d76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index d75b2c3..f9a6bc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -140,13 +140,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { // Inform the container rmContainer.handle( - new RMContainerFinishedEvent( - containerId, - containerStatus, - event) - ); - LOG.info("Completed container: " + rmContainer.getContainerId() + - " in state: " + rmContainer.getState() + " event:" + event); + new RMContainerFinishedEvent(containerId, containerStatus, event)); containersToPreempt.remove(rmContainer.getContainerId());
