Author: acmurthy
Date: Wed Nov 28 15:17:21 2012
New Revision: 1414750
URL: http://svn.apache.org/viewvc?rev=1414750&view=rev
Log:
Rebased on trunk.
Added:
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
Modified:
hadoop/common/branches/MR-2454/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
Modified: hadoop/common/branches/MR-2454/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-yarn-project/CHANGES.txt?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/MR-2454/hadoop-yarn-project/CHANGES.txt Wed Nov 28
15:17:21 2012
@@ -106,6 +106,15 @@ Release 2.0.3-alpha - Unreleased
YARN-184. Remove unnecessary locking in fair scheduler, and address
findbugs excludes. (sandyr via tucu)
+ YARN-224. Fair scheduler logs too many nodeUpdate INFO messages.
+ (Sandy Ryza via tomwhite)
+
+ YARN-222. Fair scheduler should create queue for each user by default.
+ (Sandy Ryza via tomwhite)
+
+ MAPREDUCE-4778. Fair scheduler event log is only written if directory
+ exists on HDFS. (Sandy Ryza via tomwhite)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -180,6 +189,9 @@ Release 0.23.6 - UNRELEASED
YARN-151. Browser thinks RM main page JS is taking too long
(Ravi Prakash via bobby)
+ YARN-204. test coverage for org.apache.hadoop.tools (Aleksey Gorshkov via
+ bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Wed Nov 28 15:17:21 2012
@@ -723,7 +723,9 @@ public class FairScheduler implements Re
private synchronized void nodeUpdate(RMNode nm,
List<ContainerStatus> newlyLaunchedContainers,
List<ContainerStatus> completedContainers) {
- LOG.info("nodeUpdate: " + nm + " cluster capacity: " + clusterCapacity);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterCapacity);
+ }
eventLog.log("HEARTBEAT", nm.getHostName());
FSSchedulerNode node = nodes.get(nm.getNodeID());
Modified:
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
Wed Nov 28 15:17:21 2012
@@ -18,7 +18,7 @@ public class FairSchedulerConfiguration
/** Whether to use the user name as the queue name (instead of "default") if
* the request does not specify a queue. */
protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX +
"user-as-default-queue";
- protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = false;
+ protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true;
protected static final String LOCALITY_THRESHOLD = CONF_PREFIX +
"locality.threshold";
protected static final float DEFAULT_LOCALITY_THRESHOLD = -1.0f;
Modified:
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
Wed Nov 28 15:17:21 2012
@@ -77,12 +77,11 @@ class FairSchedulerEventLog {
boolean init(FairSchedulerConfiguration conf) {
try {
logDir = conf.getEventlogDir();
- Path logDirPath = new Path(logDir);
- FileSystem fs = logDirPath.getFileSystem(conf);
- if (!fs.exists(logDirPath)) {
- if (!fs.mkdirs(logDirPath)) {
+ File logDirFile = new File(logDir);
+ if (!logDirFile.exists()) {
+ if (!logDirFile.mkdirs()) {
throw new IOException(
- "Mkdirs failed to create " + logDirPath.toString());
+ "Mkdirs failed to create " + logDirFile.toString());
}
}
String username = System.getProperty("user.name");
@@ -142,4 +141,8 @@ class FairSchedulerEventLog {
synchronized boolean isEnabled() {
return !logDisabled;
}
+
+ public String getLogFile() {
+ return logFile;
+ }
}
Modified:
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Wed Nov 28 15:17:21 2012
@@ -401,11 +401,11 @@ public class TestFairScheduler {
createAppAttemptId(1, 1), "default", "user1");
scheduler.handle(appAddedEvent1);
- // Scheduler should have one queue (the default)
- assertEquals(1, scheduler.getQueueManager().getQueues().size());
+ // Scheduler should have two queues (the default and the one created for
user1)
+ assertEquals(2, scheduler.getQueueManager().getQueues().size());
// That queue should have one app
- assertEquals(1,
scheduler.getQueueManager().getQueue("default").getApplications().size());
+ assertEquals(1,
scheduler.getQueueManager().getQueue("user1").getApplications().size());
AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);
@@ -413,8 +413,8 @@ public class TestFairScheduler {
// Now remove app
scheduler.handle(appRemovedEvent1);
- // Default queue should have no apps
- assertEquals(0,
scheduler.getQueueManager().getQueue("default").getApplications().size());
+ // Queue should have no apps
+ assertEquals(0,
scheduler.getQueueManager().getQueue("user1").getApplications().size());
}
@Test
Modified:
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
(original)
+++
hadoop/common/branches/MR-2454/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
Wed Nov 28 15:17:21 2012
@@ -56,6 +56,8 @@ History Server REST API's.
*---------------+--------------+-------------------------------+
|| Item || Data Type || Description |
*---------------+--------------+-------------------------------+
+| startedOn | long | The time the history server was started (in ms since
epoch)|
+*---------------+--------------+-------------------------------+
| hadoopVersion | string | Version of hadoop common |
*---------------+--------------+-------------------------------+
| hadoopBuildVersion | string | Hadoop common build string with build
version, user, and checksum |
@@ -87,6 +89,7 @@ History Server REST API's.
+---+
{
"historyInfo" : {
+ "startedOn":1353512830963,
"hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012",
"hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source
checksum bb6e554c6d50b0397d826081017437a7",
"hadoopVersion" : "0.23.1-SNAPSHOT"
@@ -117,6 +120,7 @@ History Server REST API's.
+---+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<historyInfo>
+ <startedOn>1353512830963</startedOn>
<hadoopVersion>0.23.1-SNAPSHOT</hadoopVersion>
<hadoopBuildVersion>0.23.1-SNAPSHOT from 1230253 by user1 source checksum
bb6e554c6d50b0397d826081017437a7</hadoopBuildVersion>
<hadoopVersionBuiltOn>Wed Jan 11 21:18:36 UTC 2012</hadoopVersionBuiltOn>
