Author: cnauroth
Date: Thu Jan 30 01:55:14 2014
New Revision: 1562670
URL: http://svn.apache.org/r1562670
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
- copied unchanged from r1562668,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
Propchange: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1561770-1562668
Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt Thu
Jan 30 01:55:14 2014
@@ -206,6 +206,12 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5725. Make explicit that TestNetworkedJob relies on the Capacity
Scheduler (Sandy Ryza)
+ MAPREDUCE-5464. Add analogs of the SLOTS_MILLIS counters that jive with the
+ YARN resource model (Sandy Ryza)
+
+ MAPREDUCE-5732. Report proper queue when job has been automatically placed
+ (Sandy Ryza)
+
OPTIMIZATIONS
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1561770-1562668
Propchange: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1561770-1562668
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Thu Jan 30 01:55:14 2014
@@ -525,6 +525,12 @@ public class JobHistoryEventHandler exte
JobInitedEvent jie = (JobInitedEvent) event.getHistoryEvent();
mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime());
}
+
+ if (event.getHistoryEvent().getEventType() ==
EventType.JOB_QUEUE_CHANGED) {
+ JobQueueChangeEvent jQueueEvent =
+ (JobQueueChangeEvent) event.getHistoryEvent();
+ mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName());
+ }
// If this is JobFinishedEvent, close the writer and setup the job-index
if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
Thu Jan 30 01:55:14 2014
@@ -39,7 +39,7 @@ import org.apache.hadoop.security.author
/**
- * Main interface to interact with the job. Provides only getters.
+ * Main interface to interact with the job.
*/
public interface Job {
@@ -98,4 +98,6 @@ public interface Job {
List<AMInfo> getAMInfos();
boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation);
+
+ public void setQueueName(String queueName);
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Thu Jan 30 01:55:14 2014
@@ -59,6 +59,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobInitedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobQueueChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletionEvent;
import org.apache.hadoop.mapreduce.lib.chain.ChainMapper;
@@ -181,7 +182,7 @@ public class JobImpl implements org.apac
private final EventHandler eventHandler;
private final MRAppMetrics metrics;
private final String userName;
- private final String queueName;
+ private String queueName;
private final long appSubmitTime;
private final AppContext appContext;
@@ -1123,6 +1124,13 @@ public class JobImpl implements org.apac
return queueName;
}
+ @Override
+ public void setQueueName(String queueName) {
+ this.queueName = queueName;
+ JobQueueChangeEvent jqce = new JobQueueChangeEvent(oldJobId, queueName);
+ eventHandler.handle(new JobHistoryEvent(jobId, jqce));
+ }
+
/*
* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.v2.app.job.Job#getConfFile()
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Thu Jan 30 01:55:14 2014
@@ -1265,57 +1265,56 @@ public abstract class TaskAttemptImpl im
}
}
}
-
- private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
+
+ private static void updateMillisCounters(JobCounterUpdateEvent jce,
+ TaskAttemptImpl taskAttempt) {
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
- int slotMemoryReq =
+ long duration = (taskAttempt.getFinishTime() -
taskAttempt.getLaunchTime());
+ int mbRequired =
taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
+ int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf,
taskType);
int minSlotMemSize = taskAttempt.conf.getInt(
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
int simSlotsRequired =
- minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq
+ minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
/ minSlotMemSize);
- long slotMillisIncrement =
- simSlotsRequired
- * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
- return slotMillisIncrement;
+ if (taskType == TaskType.MAP) {
+ jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired *
duration);
+ jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
+ jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration *
vcoresRequired);
+ jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
+ } else {
+ jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired *
duration);
+ jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration *
mbRequired);
+ jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration *
vcoresRequired);
+ jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
+ }
}
private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded(
TaskAttemptImpl taskAttempt) {
- long slotMillis = computeSlotMillis(taskAttempt);
TaskId taskId = taskAttempt.attemptId.getTaskId();
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
- jce.addCounterUpdate(
- taskId.getTaskType() == TaskType.MAP ?
- JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES,
- slotMillis);
+ updateMillisCounters(jce, taskAttempt);
return jce;
}
-
+
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
JobCounterUpdateEvent jce = new
JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
- long slotMillisIncrement = computeSlotMillis(taskAttempt);
-
if (taskType == TaskType.MAP) {
jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS,
slotMillisIncrement);
- }
} else {
jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES,
slotMillisIncrement);
- }
+ }
+ if (!taskAlreadyCompleted) {
+ updateMillisCounters(jce, taskAttempt);
}
return jce;
}
@@ -1325,20 +1324,13 @@ public abstract class TaskAttemptImpl im
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
JobCounterUpdateEvent jce = new
JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
- long slotMillisIncrement = computeSlotMillis(taskAttempt);
-
if (taskType == TaskType.MAP) {
jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS,
slotMillisIncrement);
- }
} else {
jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES,
slotMillisIncrement);
- }
+ }
+ if (!taskAlreadyCompleted) {
+ updateMillisCounters(jce, taskAttempt);
}
return jce;
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
Thu Jan 30 01:55:14 2014
@@ -109,11 +109,11 @@ public abstract class RMCommunicator ext
@Override
protected void serviceStart() throws Exception {
scheduler= createSchedulerProxy();
- register();
- startAllocatorThread();
JobID id = TypeConverter.fromYarn(this.applicationId);
JobId jobId = TypeConverter.toYarn(id);
job = context.getJob(jobId);
+ register();
+ startAllocatorThread();
super.serviceStart();
}
@@ -161,6 +161,9 @@ public abstract class RMCommunicator ext
}
this.applicationACLs = response.getApplicationACLs();
LOG.info("maxContainerCapability: " +
maxContainerCapability.getMemory());
+ String queue = response.getQueue();
+ LOG.info("queue: " + queue);
+ job.setQueueName(queue);
} catch (Exception are) {
LOG.error("Exception while registering", are);
throw new YarnRuntimeException(are);
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
Thu Jan 30 01:55:14 2014
@@ -81,6 +81,15 @@ public class TestEvents {
assertEquals(test.getPriority(), JobPriority.LOW);
}
+
+ @Test(timeout = 10000)
+ public void testJobQueueChange() throws Exception {
+ org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1);
+ JobQueueChangeEvent test = new JobQueueChangeEvent(jid,
+ "newqueue");
+ assertEquals(test.getJobId().toString(), jid.toString());
+ assertEquals(test.getJobQueueName(), "newqueue");
+ }
/**
* simple test TaskUpdatedEvent and TaskUpdated
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
Thu Jan 30 01:55:14 2014
@@ -117,6 +117,9 @@ public class MRApp extends MRAppMaster {
private File testWorkDir;
private Path testAbsPath;
private ClusterInfo clusterInfo;
+
+ // Queue to pretend the RM assigned us
+ private String assignedQueue;
public static String NM_HOST = "localhost";
public static int NM_PORT = 1234;
@@ -133,7 +136,7 @@ public class MRApp extends MRAppMaster {
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, Clock clock) {
- this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock);
+ this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock, null);
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
@@ -146,6 +149,12 @@ public class MRApp extends MRAppMaster {
boolean cleanOnStart) {
this(maps, reduces, autoComplete, testName, cleanOnStart, 1);
}
+
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, String assignedQueue) {
+ this(maps, reduces, autoComplete, testName, cleanOnStart, 1,
+ new SystemClock(), assignedQueue);
+ }
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, boolean unregistered) {
@@ -178,7 +187,7 @@ public class MRApp extends MRAppMaster {
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount) {
this(maps, reduces, autoComplete, testName, cleanOnStart, startCount,
- new SystemClock());
+ new SystemClock(), null);
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
@@ -191,33 +200,34 @@ public class MRApp extends MRAppMaster {
boolean cleanOnStart, int startCount, Clock clock, boolean unregistered)
{
this(getApplicationAttemptId(applicationId, startCount), getContainerId(
applicationId, startCount), maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, clock, unregistered);
+ cleanOnStart, startCount, clock, unregistered, null);
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
- boolean cleanOnStart, int startCount, Clock clock) {
+ boolean cleanOnStart, int startCount, Clock clock, String assignedQueue)
{
this(getApplicationAttemptId(applicationId, startCount), getContainerId(
applicationId, startCount), maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, clock, true);
+ cleanOnStart, startCount, clock, true, assignedQueue);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount, boolean unregistered) {
this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, new SystemClock(), unregistered);
+ cleanOnStart, startCount, new SystemClock(), unregistered, null);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount) {
this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, new SystemClock(), true);
+ cleanOnStart, startCount, new SystemClock(), true, null);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
- boolean cleanOnStart, int startCount, Clock clock, boolean unregistered)
{
+ boolean cleanOnStart, int startCount, Clock clock, boolean unregistered,
+ String assignedQueue) {
super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock,
System
.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
this.testWorkDir = new File("target", testName);
@@ -239,6 +249,7 @@ public class MRApp extends MRAppMaster {
// If safeToReportTerminationToUser is set to true, we can verify whether
// the job can reaches the final state when MRAppMaster shuts down.
this.successfullyUnregistered.set(unregistered);
+ this.assignedQueue = assignedQueue;
}
@Override
@@ -285,6 +296,9 @@ public class MRApp extends MRAppMaster {
start();
DefaultMetricsSystem.shutdown();
Job job = getContext().getAllJobs().values().iterator().next();
+ if (assignedQueue != null) {
+ job.setQueueName(assignedQueue);
+ }
// Write job.xml
String jobFile = MRApps.getJobFile(conf, user,
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
Thu Jan 30 01:55:14 2014
@@ -39,6 +39,7 @@ public class MockAppContext implements A
final Map<JobId, Job> jobs;
final long startTime = System.currentTimeMillis();
Set<String> blacklistedNodes;
+ String queue;
public MockAppContext(int appid) {
appID = MockJobs.newAppID(appid);
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
Thu Jan 30 01:55:14 2014
@@ -629,6 +629,11 @@ public class MockJobs extends MockApps {
jobConf.addResource(fc.open(configFile), configFile.toString());
return jobConf;
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ // do nothing
+ }
};
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
Thu Jan 30 01:55:14 2014
@@ -37,7 +37,7 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -199,7 +199,7 @@ public class TestJobEndNotifier extends
@Test
public void testNotificationOnLastRetryNormalShutdown() throws Exception {
- HttpServer server = startHttpServer();
+ HttpServer2 server = startHttpServer();
// Act like it is the second attempt. Default max attempts is 2
MRApp app = spy(new MRAppWithCustomContainerAllocator(
2, 2, true, this.getClass().getName(), true, 2, true));
@@ -223,7 +223,7 @@ public class TestJobEndNotifier extends
@Test
public void testAbsentNotificationOnNotLastRetryUnregistrationFailure()
throws Exception {
- HttpServer server = startHttpServer();
+ HttpServer2 server = startHttpServer();
MRApp app = spy(new MRAppWithCustomContainerAllocator(2, 2, false,
this.getClass().getName(), true, 1, false));
doNothing().when(app).sysexit();
@@ -250,7 +250,7 @@ public class TestJobEndNotifier extends
@Test
public void testNotificationOnLastRetryUnregistrationFailure()
throws Exception {
- HttpServer server = startHttpServer();
+ HttpServer2 server = startHttpServer();
MRApp app = spy(new MRAppWithCustomContainerAllocator(2, 2, false,
this.getClass().getName(), true, 2, false));
doNothing().when(app).sysexit();
@@ -274,10 +274,10 @@ public class TestJobEndNotifier extends
server.stop();
}
- private static HttpServer startHttpServer() throws Exception {
+ private static HttpServer2 startHttpServer() throws Exception {
new File(System.getProperty(
"build.webapps", "build/webapps") + "/test").mkdirs();
- HttpServer server = new HttpServer.Builder().setName("test")
+ HttpServer2 server = new HttpServer2.Builder().setName("test")
.addEndpoint(URI.create("http://localhost:0";))
.setFindPort(true).build();
server.addServlet("jobend", "/jobend", JobEndServlet.class);
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
Thu Jan 30 01:55:14 2014
@@ -505,6 +505,11 @@ public class TestRuntimeEstimators {
public Configuration loadConfFile() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ // do nothing
+ }
}
/*
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
Thu Jan 30 01:55:14 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapTaskAttemptImpl;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
@@ -182,13 +183,13 @@ public class TestTaskAttempt{
}
@Test
- public void testSlotMillisCounterUpdate() throws Exception {
- verifySlotMillis(2048, 2048, 1024);
- verifySlotMillis(2048, 1024, 1024);
- verifySlotMillis(10240, 1024, 2048);
+ public void testMillisCountersUpdate() throws Exception {
+ verifyMillisCounters(2048, 2048, 1024);
+ verifyMillisCounters(2048, 1024, 1024);
+ verifyMillisCounters(10240, 1024, 2048);
}
- public void verifySlotMillis(int mapMemMb, int reduceMemMb,
+ public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
int minContainerSize) throws Exception {
Clock actualClock = new SystemClock();
ControlledClock clock = new ControlledClock(actualClock);
@@ -232,13 +233,23 @@ public class TestTaskAttempt{
Assert.assertEquals(mta.getLaunchTime(), 10);
Assert.assertEquals(rta.getFinishTime(), 11);
Assert.assertEquals(rta.getLaunchTime(), 10);
+ Counters counters = job.getAllCounters();
Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
- job.getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_MAPS)
- .getValue());
- Assert.assertEquals(
- (int) Math.ceil((float) reduceMemMb / minContainerSize), job
- .getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_REDUCES)
- .getValue());
+ counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
+ Assert.assertEquals((int) Math.ceil((float) reduceMemMb /
minContainerSize),
+ counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
+ Assert.assertEquals(mapMemMb,
+ counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
+ Assert.assertEquals(reduceMemMb,
+ counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
private TaskAttemptImpl createMapTaskAttemptImplForTest(
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
Thu Jan 30 01:55:14 2014
@@ -122,6 +122,13 @@
]
},
+ {"type": "record", "name": "JobQueueChange",
+ "fields": [
+ {"name": "jobid", "type": "string"},
+ {"name": "jobQueueName", "type": "string"}
+ ]
+ },
+
{"type": "record", "name": "JobUnsuccessfulCompletion",
"fields": [
{"name": "jobid", "type": "string"},
@@ -267,6 +274,7 @@
"JOB_FINISHED",
"JOB_PRIORITY_CHANGED",
"JOB_STATUS_CHANGED",
+ "JOB_QUEUE_CHANGED",
"JOB_FAILED",
"JOB_KILLED",
"JOB_ERROR",
@@ -306,6 +314,7 @@
"JobInited",
"AMStarted",
"JobPriorityChange",
+ "JobQueueChange",
"JobStatusChanged",
"JobSubmitted",
"JobUnsuccessfulCompletion",
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
Thu Jan 30 01:55:14 2014
@@ -49,5 +49,11 @@ public enum JobCounter {
TASKS_REQ_PREEMPT,
CHECKPOINTS,
CHECKPOINT_BYTES,
- CHECKPOINT_TIME
+ CHECKPOINT_TIME,
+ MILLIS_MAPS,
+ MILLIS_REDUCES,
+ VCORES_MILLIS_MAPS,
+ VCORES_MILLIS_REDUCES,
+ MB_MILLIS_MAPS,
+ MB_MILLIS_REDUCES
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
Thu Jan 30 01:55:14 2014
@@ -98,6 +98,8 @@ public class EventReader implements Clos
result = new JobFinishedEvent(); break;
case JOB_PRIORITY_CHANGED:
result = new JobPriorityChangeEvent(); break;
+ case JOB_QUEUE_CHANGED:
+ result = new JobQueueChangeEvent(); break;
case JOB_STATUS_CHANGED:
result = new JobStatusChangedEvent(); break;
case JOB_FAILED:
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Thu Jan 30 01:55:14 2014
@@ -183,6 +183,9 @@ public class JobHistoryParser implements
case JOB_PRIORITY_CHANGED:
handleJobPriorityChangeEvent((JobPriorityChangeEvent) event);
break;
+ case JOB_QUEUE_CHANGED:
+ handleJobQueueChangeEvent((JobQueueChangeEvent) event);
+ break;
case JOB_FAILED:
case JOB_KILLED:
case JOB_ERROR:
@@ -385,6 +388,10 @@ public class JobHistoryParser implements
private void handleJobPriorityChangeEvent(JobPriorityChangeEvent event) {
info.priority = event.getPriority();
}
+
+ private void handleJobQueueChangeEvent(JobQueueChangeEvent event) {
+ info.jobQueueName = event.getJobQueueName();
+ }
private void handleJobInitedEvent(JobInitedEvent event) {
info.launchTime = event.getLaunchTime();
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1561770-1562668
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
Thu Jan 30 01:55:14 2014
@@ -25,9 +25,15 @@ DATA_LOCAL_MAPS.name= Data-
RACK_LOCAL_MAPS.name= Rack-local map tasks
SLOTS_MILLIS_MAPS.name= Total time spent by all maps in occupied
slots (ms)
SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied
slots (ms)
+MILLIS_MAPS.name= Total time spent by all map tasks (ms)
+MILLIS_REDUCES.name= Total time spent by all reduce tasks (ms)
+MB_MILLIS_MAPS.name= Total megabyte-seconds taken by all map
tasks
+MB_MILLIS_REDUCES.name= Total megabyte-seconds taken by all reduce
tasks
+VCORES_MILLIS_MAPS.name= Total vcore-seconds taken by all map tasks
+VCORES_MILLIS_REDUCES.name= Total vcore-seconds taken by all reduce
tasks
FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after
reserving slots (ms)
FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting
after reserving slots (ms)
TASKS_REQ_PREEMPT.name= Tasks that have been asked to preempt
CHECKPOINTS.name= Number of checkpoints reported
CHECKPOINT_BYTES.name= Total amount of bytes in checkpoints
-CHECKPOINT_TIME.name= Total time spent checkpointing (ms)
\ No newline at end of file
+CHECKPOINT_TIME.name= Total time spent checkpointing (ms)
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
Thu Jan 30 01:55:14 2014
@@ -34,10 +34,10 @@ import javax.servlet.http.HttpServletRes
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
public class TestJobEndNotifier extends TestCase {
- HttpServer server;
+ HttpServer2 server;
URL baseUrl;
@SuppressWarnings("serial")
@@ -102,7 +102,7 @@ public class TestJobEndNotifier extends
public void setUp() throws Exception {
new File(System.getProperty("build.webapps", "build/webapps") + "/test"
).mkdirs();
- server = new HttpServer.Builder().setName("test")
+ server = new HttpServer2.Builder().setName("test")
.addEndpoint(URI.create("http://localhost:0";))
.setFindPort(true).build();
server.addServlet("delay", "/delay", DelayServlet.class);
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
Thu Jan 30 01:55:14 2014
@@ -453,4 +453,9 @@ public class CompletedJob implements org
}
return amInfos;
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ throw new UnsupportedOperationException("Can't set job's queue name in
history");
+ }
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
Thu Jan 30 01:55:14 2014
@@ -190,5 +190,10 @@ public class PartialJob implements org.a
public List<AMInfo> getAMInfos() {
return null;
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ throw new UnsupportedOperationException("Can't set job's queue name in
history");
+ }
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
Thu Jan 30 01:55:14 2014
@@ -155,6 +155,41 @@ public class TestJobHistoryEvents {
Assert.assertEquals("JobHistoryEventHandler",
services[services.length - 1].getName());
}
+
+ @Test
+ public void testAssignedQueue() throws Exception {
+ Configuration conf = new Configuration();
+ MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(),
+ true, "assignedQueue");
+ app.submit(conf);
+ Job job = app.getContext().getAllJobs().values().iterator().next();
+ JobId jobId = job.getID();
+ LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
+ app.waitForState(job, JobState.SUCCEEDED);
+
+ //make sure all events are flushed
+ app.waitForState(Service.STATE.STOPPED);
+ /*
+ * Use HistoryContext to read logged events and verify the number of
+ * completed maps
+ */
+ HistoryContext context = new JobHistory();
+ // test start and stop states
+ ((JobHistory)context).init(conf);
+ ((JobHistory)context).start();
+ Assert.assertTrue( context.getStartTime()>0);
+
Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);
+
+ // get job before stopping JobHistory
+ Job parsedJob = context.getJob(jobId);
+
+ // stop JobHistory
+ ((JobHistory)context).stop();
+
Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);
+
+ Assert.assertEquals("QueueName not correct", "assignedQueue",
+ parsedJob.getQueueName());
+ }
private void verifyTask(Task task) {
Assert.assertEquals("Task state not currect", TaskState.SUCCEEDED,
@@ -184,6 +219,11 @@ public class TestJobHistoryEvents {
super(maps, reduces, autoComplete, testName, cleanOnStart);
}
+ public MRAppWithHistory(int maps, int reduces, boolean autoComplete,
+ String testName, boolean cleanOnStart, String assignedQueue) {
+ super(maps, reduces, autoComplete, testName, cleanOnStart,
assignedQueue);
+ }
+
@Override
protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
AppContext context) {
Modified:
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
Thu Jan 30 01:55:14 2014
@@ -415,5 +415,9 @@ public class TestHsWebServicesAcls {
return aclsMgr.checkAccess(callerUGI, jobOperation,
this.getUserName(), jobAcls.get(jobOperation));
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ }
}
}
