Author: sandy
Date: Fri Dec 6 01:34:42 2013
New Revision: 1548362
URL: http://svn.apache.org/r1548362
Log:
YARN-546. Allow disabling the Fair Scheduler event log. (Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1548362&r1=1548361&r2=1548362&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Dec 6
01:34:42 2013
@@ -125,6 +125,8 @@ Release 2.4.0 - UNRELEASED
YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla)
+ YARN-546. Allow disabling the Fair Scheduler event log (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1548362&r1=1548361&r2=1548362&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
Fri Dec 6 01:34:42 2013
@@ -53,6 +53,11 @@ public class FairSchedulerConfiguration
public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
+
+ /** Whether to enable the Fair Scheduler event log */
+ public static final String EVENT_LOG_ENABLED = CONF_PREFIX +
"event-log-enabled";
+ public static final boolean DEFAULT_EVENT_LOG_ENABLED = false;
+
protected static final String EVENT_LOG_DIR = "eventlog.dir";
/** Whether pools can be created that were not specified in the FS
configuration file
@@ -192,6 +197,10 @@ public class FairSchedulerConfiguration
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
}
+ public boolean isEventLogEnabled() {
+ return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED);
+ }
+
public String getEventlogDir() {
return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir",
"/tmp/")).getAbsolutePath() + File.separator + "fairscheduler");
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java?rev=1548362&r1=1548361&r2=1548362&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
Fri Dec 6 01:34:42 2013
@@ -75,26 +75,30 @@ class FairSchedulerEventLog {
private DailyRollingFileAppender appender;
boolean init(FairSchedulerConfiguration conf) {
- try {
- logDir = conf.getEventlogDir();
- File logDirFile = new File(logDir);
- if (!logDirFile.exists()) {
- if (!logDirFile.mkdirs()) {
- throw new IOException(
- "Mkdirs failed to create " + logDirFile.toString());
+ if (conf.isEventLogEnabled()) {
+ try {
+ logDir = conf.getEventlogDir();
+ File logDirFile = new File(logDir);
+ if (!logDirFile.exists()) {
+ if (!logDirFile.mkdirs()) {
+ throw new IOException(
+ "Mkdirs failed to create " + logDirFile.toString());
+ }
}
+ String username = System.getProperty("user.name");
+ logFile = String.format("%s%shadoop-%s-fairscheduler.log",
+ logDir, File.separator, username);
+ logDisabled = false;
+ PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
+ appender = new DailyRollingFileAppender(layout, logFile,
"'.'yyyy-MM-dd");
+ appender.activateOptions();
+ LOG.info("Initialized fair scheduler event log, logging to " +
logFile);
+ } catch (IOException e) {
+ LOG.error(
+ "Failed to initialize fair scheduler event log. Disabling it.", e);
+ logDisabled = true;
}
- String username = System.getProperty("user.name");
- logFile = String.format("%s%shadoop-%s-fairscheduler.log",
- logDir, File.separator, username);
- logDisabled = false;
- PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
- appender = new DailyRollingFileAppender(layout, logFile,
"'.'yyyy-MM-dd");
- appender.activateOptions();
- LOG.info("Initialized fair scheduler event log, logging to " + logFile);
- } catch (IOException e) {
- LOG.error(
- "Failed to initialize fair scheduler event log. Disabling it.", e);
+ } else {
logDisabled = true;
}
return !(logDisabled);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java?rev=1548362&r1=1548361&r2=1548362&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
Fri Dec 6 01:34:42 2013
@@ -44,7 +44,7 @@ public class TestFairSchedulerEventLog {
Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
- conf.set("mapred.fairscheduler.eventlog.enabled", "true");
+ conf.set("yarn.scheduler.fair.event-log-enabled", "true");
// All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
