Author: cnauroth
Date: Sat Jan 18 06:50:19 2014
New Revision: 1559324
URL: http://svn.apache.org/r1559324
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
- copied unchanged from r1559323,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
- copied unchanged from r1559323,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Propchange: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1557294-1559323
Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt Sat
Jan 18 06:50:19 2014
@@ -146,6 +146,8 @@ Trunk (Unreleased)
MAPREDUCE-5191. TestQueue#testQueue fails with timeout on Windows. (Ivan
Mitic via hitesh)
+ MAPREDUCE-5717. Task pings are interpreted as task progress (jlowe)
+
Release 2.4.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -198,6 +200,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-3310. Custom grouping comparator cannot be set for Combiners
(tucu)
+ MAPREDUCE-5672. Provide optional RollingFileAppender for container log4j
+ (syslog) (Gera Shegalov via jlowe)
+
OPTIMIZATIONS
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
@@ -272,6 +277,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5689. MRAppMaster does not preempt reducers when scheduled maps
cannot be fulfilled. (lohit via kasha)
+ MAPREDUCE-5724. JobHistoryServer does not start if HDFS is not running.
+ (tucu)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -328,6 +336,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus.
(Chuan Liu via cnauroth)
+ MAPREDUCE-5650. Job fails when hprof
mapreduce.task.profile.map/reduce.params
+ is specified (Gera Shegalov via Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
@@ -998,6 +1009,16 @@ Release 2.1.0-beta - 2013-08-22
MAPREDUCE-4374. Fix child task environment variable config and add support
for Windows. (Chuan Liu via cnauroth)
+Release 2.0.6-alpha - 08/22/2013
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1557294-1559323
Propchange: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1557294-1559323
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
Sat Jan 18 06:50:19 2014
@@ -149,8 +149,10 @@ public class MapReduceChildJVM {
private static void setupLog4jProperties(Task task,
Vector<String> vargs,
long logSize) {
- String logLevel = getChildLogLevel(task.conf, task.isMapTask());
- MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+ String logLevel = getChildLogLevel(task.conf, task.isMapTask());
+ int numBackups = task.conf.getInt(MRJobConfig.TASK_LOG_BACKUPS,
+ MRJobConfig.DEFAULT_TASK_LOG_BACKUPS);
+ MRApps.addLog4jSystemProperties(logLevel, logSize, numBackups, vargs);
}
public static List<String> getVMCommand(
@@ -210,19 +212,11 @@ public class MapReduceChildJVM {
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(task.isMapTask()
).isIncluded(task.getPartition())) {
- vargs.add(
- String.format(
- conf.getProfileParams(),
- getTaskLogFile(TaskLog.LogName.PROFILE)
- )
- );
- if (task.isMapTask()) {
- vargs.add(conf.get(MRJobConfig.TASK_MAP_PROFILE_PARAMS, ""));
- }
- else {
- vargs.add(conf.get(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, ""));
- }
-
+ final String profileParams = conf.get(task.isMapTask()
+ ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
+ : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
+ vargs.add(String.format(profileParams,
+ getTaskLogFile(TaskLog.LogName.PROFILE)));
}
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
Sat Jan 18 06:50:19 2014
@@ -361,7 +361,6 @@ public class TaskAttemptListenerImpl ext
if (taskStatus == null) {
//We are using statusUpdate only as a simple ping
LOG.info("Ping from " + taskAttemptID.toString());
- taskHeartbeatHandler.progressing(yarnAttemptID);
return feedback;
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
Sat Jan 18 06:50:19 2014
@@ -381,4 +381,50 @@ public class TestTaskAttemptListenerImpl
}
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testStatusUpdateProgress()
+ throws IOException, InterruptedException {
+ AppContext appCtx = mock(AppContext.class);
+ JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
+ RMHeartbeatHandler rmHeartbeatHandler =
+ mock(RMHeartbeatHandler.class);
+ TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
+ Dispatcher dispatcher = mock(Dispatcher.class);
+ EventHandler ea = mock(EventHandler.class);
+ when(dispatcher.getEventHandler()).thenReturn(ea);
+
+ when(appCtx.getEventHandler()).thenReturn(ea);
+ CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
+ policy.init(appCtx);
+ MockTaskAttemptListenerImpl listener =
+ new MockTaskAttemptListenerImpl(appCtx, secret,
+ rmHeartbeatHandler, hbHandler, policy);
+ Configuration conf = new Configuration();
+ listener.init(conf);
+ listener.start();
+ JVMId id = new JVMId("foo",1, true, 1);
+ WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId());
+
+ TaskAttemptID attemptID = new TaskAttemptID("1", 1, TaskType.MAP, 1, 1);
+ TaskAttemptId attemptId = TypeConverter.toYarn(attemptID);
+ Task task = mock(Task.class);
+ listener.registerPendingTask(task, wid);
+ listener.registerLaunchedTask(attemptId, wid);
+ verify(hbHandler).register(attemptId);
+
+ // make sure a ping doesn't report progress
+ AMFeedback feedback = listener.statusUpdate(attemptID, null);
+ assertTrue(feedback.getTaskFound());
+ verify(hbHandler, never()).progressing(eq(attemptId));
+
+ // make sure a status update does report progress
+ MapTaskStatus mockStatus = new MapTaskStatus(attemptID, 0.0f, 1,
+ TaskStatus.State.RUNNING, "", "RUNNING", "", TaskStatus.Phase.MAP,
+ new Counters());
+ feedback = listener.statusUpdate(attemptID, mockStatus);
+ assertTrue(feedback.getTaskFound());
+ verify(hbHandler).progressing(eq(attemptId));
+ listener.close();
+ }
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
Sat Jan 18 06:50:19 2014
@@ -78,6 +78,13 @@ public class JHAdminConfig {
MR_HISTORY_PREFIX + "done-dir";
/**
+ * Maximum time the History server will wait for the FileSystem for History
+ * files to become available. Default value is -1, forever.
+ */
+ public static final String MR_HISTORY_MAX_START_WAIT_TIME =
+ MR_HISTORY_PREFIX + "maximum-start-wait-time-millis";
+ public static final long DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME = -1;
+ /**
* Path where history files should be stored after a job finished and before
* they are pulled into the job history server.
**/
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Sat Jan 18 06:50:19 2014
@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.log4j.RollingFileAppender;
/**
* Helper class for MR applications
@@ -476,16 +477,24 @@ public class MRApps extends Apps {
* Add the JVM system properties necessary to configure {@link
ContainerLogAppender}.
* @param logLevel the desired log level (eg INFO/WARN/DEBUG)
* @param logSize See {@link ContainerLogAppender#setTotalLogFileSize(long)}
+ * @param numBackups See {@link RollingFileAppender#setMaxBackupIndex(int)}
* @param vargs the argument list to append to
*/
public static void addLog4jSystemProperties(
- String logLevel, long logSize, List<String> vargs) {
+ String logLevel, long logSize, int numBackups, List<String> vargs) {
vargs.add("-Dlog4j.configuration=container-log4j.properties");
vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add(
"-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + logSize);
- vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+ if (logSize > 0L && numBackups > 0) {
+ // log should be rolled
+ vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_BACKUPS + "="
+ + numBackups);
+ vargs.add("-Dhadoop.root.logger=" + logLevel + ",CRLA");
+ } else {
+ vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+ }
}
/**
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Sat Jan 18 06:50:19 2014
@@ -412,6 +412,10 @@ public interface MRJobConfig {
MR_AM_PREFIX+"log.level";
public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO";
+ public static final String MR_AM_LOG_BACKUPS =
+ MR_AM_PREFIX + "container.log.backups";
+ public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll
+
/**The number of splits when reporting progress in MR*/
public static final String MR_AM_NUM_PROGRESS_SPLITS =
MR_AM_PREFIX+"num-progress-splits";
@@ -692,7 +696,11 @@ public interface MRJobConfig {
+ "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*";
public static final String WORKFLOW_ID = "mapreduce.workflow.id";
-
+
+ public static final String TASK_LOG_BACKUPS =
+ MR_PREFIX + "task.container.log.backups";
+ public static final int DEFAULT_TASK_LOG_BACKUPS = 0; // don't roll
+
public static final String WORKFLOW_NAME = "mapreduce.workflow.name";
public static final String WORKFLOW_NODE_NAME =
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Sat Jan 18 06:50:19 2014
@@ -511,6 +511,31 @@
</property>
<property>
+ <name>yarn.app.mapreduce.task.container.log.backups</name>
+ <value>0</value>
+ <description>Number of backup files for task logs when using
+ ContainerRollingLogAppender (CRLA). See
+ org.apache.log4j.RollingFileAppender.maxBackupIndex. By default,
+ ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA
+ is enabled for tasks when both mapreduce.task.userlog.limit.kb and
+ yarn.app.mapreduce.task.container.log.backups are greater than zero.
+ </description>
+</property>
+
+<property>
+ <name>yarn.app.mapreduce.am.container.log.backups</name>
+ <value>0</value>
+ <description>Number of backup files for the ApplicationMaster logs when using
+ ContainerRollingLogAppender (CRLA). See
+ org.apache.log4j.RollingFileAppender.maxBackupIndex. By default,
+ ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA
+ is enabled for the ApplicationMaster when both
+ mapreduce.task.userlog.limit.kb and
+ yarn.app.mapreduce.am.container.log.backups are greater than zero.
+ </description>
+</property>
+
+<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
<value>3</value>
<description>The number of task-failures on a node manager of a given job
@@ -578,6 +603,31 @@
</property>
<property>
+ <name>mapreduce.task.profile.params</name>
+ <value></value>
+ <description>JVM profiler parameters used to profile map and reduce task
+ attempts. This string may contain a single format specifier %s that will
+ be replaced by the path to profile.out in the task attempt log directory.
+ To specify different profiling options for map tasks and reduce tasks,
+ more specific parameters mapreduce.task.profile.map.params and
+ mapreduce.task.profile.reduce.params should be used.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.map.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Map-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.reduce.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Reduce-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
<name>mapreduce.task.skip.start.attempts</name>
<value>2</value>
<description> The number of Task attempts AFTER which skip mode
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1557294-1559323
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
Sat Jan 18 06:50:19 2014
@@ -35,6 +35,10 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</dependency>
<dependency>
@@ -53,6 +57,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
Sat Jan 18 06:50:19 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.h
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.ConnectException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -69,6 +70,8 @@ import org.apache.hadoop.yarn.exceptions
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
/**
* This class provides a way to interact with history files in a thread safe
@@ -464,7 +467,8 @@ public class HistoryFileManager extends
private JobACLsManager aclsMgr;
- private Configuration conf;
+ @VisibleForTesting
+ Configuration conf;
private String serialNumberFormat;
@@ -491,36 +495,10 @@ public class HistoryFileManager extends
+ (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS +
serialNumberLowDigits)
+ "d");
- String doneDirPrefix = null;
- doneDirPrefix = JobHistoryUtils
- .getConfiguredHistoryServerDoneDirPrefix(conf);
- try {
- doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(
- new Path(doneDirPrefix));
- doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
- doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
- mkdir(doneDirFc, doneDirPrefixPath, new FsPermission(
- JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
- } catch (IOException e) {
- throw new YarnRuntimeException("Error creating done directory: ["
- + doneDirPrefixPath + "]", e);
- }
-
- String intermediateDoneDirPrefix = null;
- intermediateDoneDirPrefix = JobHistoryUtils
- .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
- try {
- intermediateDoneDirPath = FileContext.getFileContext(conf).makeQualified(
- new Path(intermediateDoneDirPrefix));
- intermediateDoneDirFc = FileContext.getFileContext(
- intermediateDoneDirPath.toUri(), conf);
- mkdir(intermediateDoneDirFc, intermediateDoneDirPath, new FsPermission(
-
JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
- } catch (IOException e) {
- LOG.info("error creating done directory on dfs " + e);
- throw new YarnRuntimeException("Error creating intermediate done
directory: ["
- + intermediateDoneDirPath + "]", e);
- }
+ long maxFSWaitTime = conf.getLong(
+ JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
+ JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
+ createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);
this.aclsMgr = new JobACLsManager(conf);
@@ -544,6 +522,107 @@ public class HistoryFileManager extends
super.serviceInit(conf);
}
+ @VisibleForTesting
+ void createHistoryDirs(Clock clock, long intervalCheckMillis,
+ long timeOutMillis) throws IOException {
+ long start = clock.getTime();
+ boolean done = false;
+ int counter = 0;
+ while (!done &&
+ ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
+ done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3
attempts, 30sec
+ try {
+ Thread.sleep(intervalCheckMillis);
+ } catch (InterruptedException ex) {
+ throw new YarnRuntimeException(ex);
+ }
+ }
+ if (!done) {
+ throw new YarnRuntimeException("Timed out '" + timeOutMillis+
+ "ms' waiting for FileSystem to become available");
+ }
+ }
+
+ /**
+ * DistributedFileSystem returns a RemoteException with a message stating
+ * SafeModeException in it. So this is only way to check it is because of
+ * being in safe mode.
+ */
+ private boolean isBecauseSafeMode(Throwable ex) {
+ return ex.toString().contains("SafeModeException");
+ }
+
+ /**
+ * Returns TRUE if the history dirs were created, FALSE if they could not
+ * be created because the FileSystem is not reachable or in safe mode and
+ * throws and exception otherwise.
+ */
+ @VisibleForTesting
+ boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
+ boolean succeeded = true;
+ String doneDirPrefix = JobHistoryUtils.
+ getConfiguredHistoryServerDoneDirPrefix(conf);
+ try {
+ doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(
+ new Path(doneDirPrefix));
+ doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
+ doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
+ mkdir(doneDirFc, doneDirPrefixPath, new FsPermission(
+ JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
+ } catch (ConnectException ex) {
+ if (logWait) {
+ LOG.info("Waiting for FileSystem at " +
+ doneDirPrefixPath.toUri().getAuthority() + "to be available");
+ }
+ succeeded = false;
+ } catch (IOException e) {
+ if (isBecauseSafeMode(e)) {
+ succeeded = false;
+ if (logWait) {
+ LOG.info("Waiting for FileSystem at " +
+ doneDirPrefixPath.toUri().getAuthority() +
+ "to be out of safe mode");
+ }
+ } else {
+ throw new YarnRuntimeException("Error creating done directory: ["
+ + doneDirPrefixPath + "]", e);
+ }
+ }
+ if (succeeded) {
+ String intermediateDoneDirPrefix = JobHistoryUtils.
+ getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+ try {
+ intermediateDoneDirPath =
FileContext.getFileContext(conf).makeQualified(
+ new Path(intermediateDoneDirPrefix));
+ intermediateDoneDirFc = FileContext.getFileContext(
+ intermediateDoneDirPath.toUri(), conf);
+ mkdir(intermediateDoneDirFc, intermediateDoneDirPath, new FsPermission(
+
JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
+ } catch (ConnectException ex) {
+ succeeded = false;
+ if (logWait) {
+ LOG.info("Waiting for FileSystem at " +
+ intermediateDoneDirPath.toUri().getAuthority() +
+ "to be available");
+ }
+ } catch (IOException e) {
+ if (isBecauseSafeMode(e)) {
+ succeeded = false;
+ if (logWait) {
+ LOG.info("Waiting for FileSystem at " +
+ intermediateDoneDirPath.toUri().getAuthority() +
+ "to be out of safe mode");
+ }
+ } else {
+ throw new YarnRuntimeException(
+ "Error creating intermediate done directory: ["
+ + intermediateDoneDirPath + "]", e);
+ }
+ }
+ }
+ return succeeded;
+ }
+
@Override
public void serviceStop() throws Exception {
ShutdownThreadsHelper.shutdownExecutorService(moveToDoneExecutor);
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Sat Jan 18 06:50:19 2014
@@ -392,7 +392,9 @@ public class YARNRunner implements Clien
long logSize = TaskLog.getTaskLogLength(new JobConf(conf));
String logLevel = jobConf.get(
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
- MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+ int numBackups = jobConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS,
+ MRJobConfig.DEFAULT_MR_AM_LOG_BACKUPS);
+ MRApps.addLog4jSystemProperties(logLevel, logSize, numBackups, vargs);
// Check for Java Lib Path usage in MAP and REDUCE configs
warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map",
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
Sat Jan 18 06:50:19 2014
@@ -90,6 +90,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import
org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
+import
org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
@@ -99,6 +101,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -354,6 +357,12 @@ public class TestClientRedirect {
CancelDelegationTokenRequest request) throws IOException {
return null;
}
+
+ @Override
+ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
+ MoveApplicationAcrossQueuesRequest request) throws YarnException,
IOException {
+ return null;
+ }
}
class HistoryService extends AMService implements HSClientProtocol {
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Sat Jan 18 06:50:19 2014
@@ -23,10 +23,12 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
+import java.io.InputStreamReader;
import java.io.IOException;
import java.io.StringReader;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.jar.JarOutputStream;
@@ -53,6 +55,8 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobCounter;
@@ -65,17 +69,22 @@ import org.apache.hadoop.mapreduce.TaskC
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskReport;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -84,6 +93,9 @@ import org.junit.Test;
public class TestMRJobs {
private static final Log LOG = LogFactory.getLog(TestMRJobs.class);
+ private static final EnumSet<RMAppState> TERMINAL_RM_APP_STATES =
+ EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);
+ private static final int NUM_NODE_MGRS = 3;
protected static MiniMRYarnCluster mrCluster;
protected static MiniDFSCluster dfsCluster;
@@ -122,7 +134,8 @@ public class TestMRJobs {
}
if (mrCluster == null) {
- mrCluster = new MiniMRYarnCluster(TestMRJobs.class.getName(), 3);
+ mrCluster = new MiniMRYarnCluster(TestMRJobs.class.getName(),
+ NUM_NODE_MGRS);
Configuration conf = new Configuration();
conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
@@ -416,6 +429,115 @@ public class TestMRJobs {
// TODO later: add explicit "isUber()" checks of some sort
}
+ @Test(timeout = 120000)
+ public void testContainerRollingLog() throws IOException,
+ InterruptedException, ClassNotFoundException {
+ if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ final SleepJob sleepJob = new SleepJob();
+ final JobConf sleepConf = new JobConf(mrCluster.getConfig());
+ sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
+ sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
+ sleepConf.setLong(MRJobConfig.TASK_USERLOG_LIMIT, 1);
+ sleepConf.setInt(MRJobConfig.TASK_LOG_BACKUPS, 3);
+ sleepConf.setInt(MRJobConfig.MR_AM_LOG_BACKUPS, 7);
+ sleepJob.setConf(sleepConf);
+
+ final Job job = sleepJob.createJob(1, 0, 1L, 100, 0L, 0);
+ job.setJarByClass(SleepJob.class);
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.waitForCompletion(true);
+ final JobId jobId = TypeConverter.toYarn(job.getJobID());
+ final ApplicationId appID = jobId.getAppId();
+ int pollElapsed = 0;
+ while (true) {
+ Thread.sleep(1000);
+ pollElapsed += 1000;
+ if (TERMINAL_RM_APP_STATES.contains(
+ mrCluster.getResourceManager().getRMContext().getRMApps().get(appID)
+ .getState())) {
+ break;
+ }
+ if (pollElapsed >= 60000) {
+ LOG.warn("application did not reach terminal state within 60 seconds");
+ break;
+ }
+ }
+ Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager()
+ .getRMContext().getRMApps().get(appID).getState());
+
+ // Job finished, verify logs
+ //
+
+ final String appIdStr = appID.toString();
+ final String appIdSuffix = appIdStr.substring("application_".length(),
+ appIdStr.length());
+ final String containerGlob = "container_" + appIdSuffix + "_*_*";
+ final String syslogGlob = appIdStr
+ + Path.SEPARATOR + containerGlob
+ + Path.SEPARATOR + TaskLog.LogName.SYSLOG;
+ int numAppMasters = 0;
+ int numMapTasks = 0;
+
+ for (int i = 0; i < NUM_NODE_MGRS; i++) {
+ final Configuration nmConf = mrCluster.getNodeManager(i).getConfig();
+ for (String logDir :
+ nmConf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)) {
+ final Path absSyslogGlob =
+ new Path(logDir + Path.SEPARATOR + syslogGlob);
+ LOG.info("Checking for glob: " + absSyslogGlob);
+ final FileStatus[] syslogs = localFs.globStatus(absSyslogGlob);
+ for (FileStatus slog : syslogs) {
+ // check all syslogs for the container
+ //
+ final FileStatus[] sysSiblings = localFs.globStatus(new Path(
+ slog.getPath().getParent(), TaskLog.LogName.SYSLOG + "*"));
+ boolean foundAppMaster = false;
+ floop:
+ for (FileStatus f : sysSiblings) {
+ final BufferedReader reader = new BufferedReader(
+ new InputStreamReader(localFs.open(f.getPath())));
+ String line;
+ try {
+ while ((line = reader.readLine()) != null) {
+ if (line.contains(MRJobConfig.APPLICATION_MASTER_CLASS)) {
+ foundAppMaster = true;
+ break floop;
+ }
+ }
+ } finally {
+ reader.close();
+ }
+ }
+
+ if (foundAppMaster) {
+ numAppMasters++;
+ } else {
+ numMapTasks++;
+ }
+
+ Assert.assertSame("Number of sylog* files",
+ foundAppMaster
+ ? sleepConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS, 0) + 1
+ : sleepConf.getInt(MRJobConfig.TASK_LOG_BACKUPS, 0) + 1,
+ sysSiblings.length);
+ }
+ }
+ }
+ // Make sure we checked non-empty set
+ //
+ Assert.assertEquals("No AppMaster log found!", 1, numAppMasters);
+ if (sleepConf.getBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false)) {
+ Assert.assertEquals("MapTask log with uber found!", 0, numMapTasks);
+ } else {
+ Assert.assertEquals("No MapTask log found!", 1, numMapTasks);
+ }
+ }
+
public static class DistributedCacheChecker extends
Mapper<LongWritable, Text, NullWritable, NullWritable> {
