Author: jlowe
Date: Wed May 29 14:25:04 2013
New Revision: 1487498
URL: http://svn.apache.org/r1487498
Log:
YARN-512. Log aggregation root directory check is more expensive than it needs
to be. Contributed by Maysam Yabandeh
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1487498&r1=1487497&r2=1487498&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed May 29 14:25:04 2013
@@ -223,6 +223,9 @@ Release 2.0.5-beta - UNRELEASED
OPTIMIZATIONS
+ YARN-512. Log aggregation root directory check is more expensive than it
+ needs to be. (Maysam Yabandeh via jlowe)
+
BUG FIXES
YARN-383. AMRMClientImpl should handle null rmClient in stop()
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1487498&r1=1487497&r2=1487498&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
Wed May 29 14:25:04 2013
@@ -18,6 +18,7 @@
package
org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Map;
@@ -163,36 +164,31 @@ public class LogAggregationService exten
}
void verifyAndCreateRemoteLogDir(Configuration conf) {
- // Checking the existance of the TLD
+ // Checking the existence of the TLD
FileSystem remoteFS = null;
try {
remoteFS = FileSystem.get(conf);
} catch (IOException e) {
throw new YarnException("Unable to get Remote FileSystem instance", e);
}
- boolean remoteExists = false;
+ boolean remoteExists = true;
try {
- remoteExists = remoteFS.exists(this.remoteRootLogDir);
+ FsPermission perms =
+ remoteFS.getFileStatus(this.remoteRootLogDir).getPermission();
+ if (!perms.equals(TLDIR_PERMISSIONS)) {
+ LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
+ + "] already exist, but with incorrect permissions. "
+ + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
+ + "]." + " The cluster may have problems with multiple users.");
+ }
+ } catch (FileNotFoundException e) {
+ remoteExists = false;
} catch (IOException e) {
- throw new YarnException("Failed to check for existence of remoteLogDir ["
- + this.remoteRootLogDir + "]", e);
+ throw new YarnException(
+ "Failed to check permissions for dir ["
+ + this.remoteRootLogDir + "]", e);
}
- if (remoteExists) {
- try {
- FsPermission perms =
- remoteFS.getFileStatus(this.remoteRootLogDir).getPermission();
- if (!perms.equals(TLDIR_PERMISSIONS)) {
- LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
- + "] already exist, but with incorrect permissions. "
- + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
- + "]." + " The cluster may have problems with multiple users.");
- }
- } catch (IOException e) {
- throw new YarnException(
- "Failed to check permissions for dir ["
- + this.remoteRootLogDir + "]", e);
- }
- } else {
+ if (!remoteExists) {
LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
+ "] does not exist. Attempting to create it.");
try {
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1487498&r1=1487497&r2=1487498&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Wed May 29 14:25:04 2013
@@ -465,6 +465,30 @@ public class TestLogAggregationService e
logAggregationService.stop();
}
+
+ @Test
+ public void testVerifyAndCreateRemoteDirNonExistence()
+ throws Exception {
+ this.conf.set(YarnConfiguration.NM_LOG_DIRS,
localLogDir.getAbsolutePath());
+ File aNewFile = new File(String.valueOf("tmp"+System.currentTimeMillis()));
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ aNewFile.getAbsolutePath());
+
+ DrainDispatcher dispatcher = createDispatcher();
+ LogAggregationService logAggregationService = spy(
+ new LogAggregationService(dispatcher, this.context, this.delSrvc,
+ super.dirsHandler));
+ logAggregationService.init(this.conf);
+ boolean existsBefore = aNewFile.exists();
+ assertTrue("The new file already exists!", !existsBefore);
+
+ logAggregationService.verifyAndCreateRemoteLogDir(this.conf);
+
+ boolean existsAfter = aNewFile.exists();
+ assertTrue("The new aggregate file is not successfully created",
existsAfter);
+ aNewFile.delete(); //housekeeping
+ }
+
@Test
@SuppressWarnings("unchecked")
public void testLogAggregationInitAppFailsWithoutKillingNM() throws
Exception {
