Author: cmccabe
Date: Fri Apr 18 16:32:35 2014
New Revision: 1588509
URL: http://svn.apache.org/r1588509
Log:
Merge trunk into the HADOOP-10388 branch
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/ (props
changed)
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/bin/mapred.cmd
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/SleepJob.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SleepJob.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
Propchange: hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1582150-1588387
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/CHANGES.txt?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/CHANGES.txt
Fri Apr 18 16:32:35 2014
@@ -68,9 +68,6 @@ Trunk (Unreleased)
MAPREDUCE-4735. Make arguments in TestDFSIO case insensitive.
(Brandon Li via suresh)
- MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing.
- (Srikanth Sundarrajan via amareshwari)
-
MAPREDUCE-5197. Add a service for checkpointing task state.
(Carlo Curino via cdouglas)
@@ -163,11 +160,51 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5804. TestMRJobsWithProfiler#testProfiler timesout (Mit Desai
via kihwal)
+ MAPREDUCE-5825. Provide diagnostics for reducers killed during ramp down
+ (Gera Shegalov via jlowe)
+
+ MAPREDUCE-5836. Fix typo in RandomTextWriter (Akira AJISAKA via jeagles)
+
OPTIMIZATIONS
BUG FIXES
-Release 2.4.0 - UNRELEASED
+ MAPREDUCE-5759. Remove unnecessary conf load in Limits (Sandy Ryza)
+
+ MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing.
+ (Srikanth Sundarrajan via amareshwari)
+
+ MAPREDUCE-5775. Remove unnecessary job.setNumReduceTasks in
SleepJob.createJob
+ (jhanver chand sharma via devaraj)
+
+Release 2.4.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ MAPREDUCE-5818. Added "hsadmin" command into mapred.cmd. (Jian He via
zjshen)
+
+ MAPREDUCE-5824. Fixed test-failure of TestPipesNonJavaInputFormat in
+ Windows. (Xuan Gong via vinodkv)
+
+ MAPREDUCE-5815. Fixed test-failure of TestMRAppMaster by making MRAppMaster
+ gracefully handle empty-queue names. (Akira Ajisaka via vinodkv)
+
+ MAPREDUCE-5826. Fixed HistoryServerFileSystemStore to use right permissions
+ on Windows for temporary files and thus also fix the test-issue with
+ TestHistoryServerFileSystemStateStoreService. (Varun Vasudev via vinodkv)
+
+ MAPREDUCE-5828. Fixed a test issue with TestMapReduceJobControl that was
+ causing it to fail on Windows. (vinodkv)
+
+Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
@@ -269,6 +306,13 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape
hyphens and thus avoid parsing errors. (Akira AJISAKA via vinodkv)
+ MAPREDUCE-5810. Removed the faulty and failing streaming test
+ TestStreamingTaskLog. (Akira Ajisaka via vinodkv)
+
+ MAPREDUCE-5813. Fix YarnChild to explicitly load job.xml from the
+ local-filesystem, rather than rely on the classpath. (Gera Shegalov via
+ acmurthy)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1740,6 +1784,9 @@ Release 0.23.10 - UNRELEASED
MAPREDUCE-5640. Rename TestLineRecordReader in jobclient module (Jason Lowe
via jeagles)
+ MAPREDUCE-3191. docs for map output compression incorrectly reference
+ SequenceFile (Chen He via jeagles)
+
OPTIMIZATIONS
MAPREDUCE-1981. Improve getSplits performance by using listLocatedStatus
Propchange:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1582150-1588387
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/bin/mapred.cmd
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/bin/mapred.cmd?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/bin/mapred.cmd
(original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/bin/mapred.cmd
Fri Apr 18 16:32:35 2014
@@ -135,6 +135,10 @@ goto :eof
set CLASSPATH=%CLASSPATH%;%TOO_PATH%
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
+:hsadmin
+ set CLASS=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
+ set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
+
:pipes
goto not_supported
@@ -189,6 +193,7 @@ goto :eof
@echo historyserver run job history servers as a standalone daemon
@echo distcp ^<srcurl^> ^<desturl^> copy file or directories recursively
@echo archive -archiveName NAME -p ^<parent path^> ^<src^>* ^<dest^>
create a hadoop archive
+ @echo hsadmin job history server admin interface
@echo
@echo Most commands print help when invoked w/o parameters.
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Fri Apr 18 16:32:35 2014
@@ -76,10 +76,9 @@ class YarnChild {
Thread.setDefaultUncaughtExceptionHandler(new
YarnUncaughtExceptionHandler());
LOG.debug("Child starting");
- final JobConf job = new JobConf();
+ final JobConf job = new JobConf(MRJobConfig.JOB_CONF_FILE);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.init(job);
- job.addResource(MRJobConfig.JOB_CONF_FILE);
UserGroupInformation.setConfiguration(job);
String host = args[0];
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Apr 18 16:32:35 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobCounter;
@@ -442,8 +443,13 @@ public class JobHistoryEventHandler exte
}
}
+ String queueName = JobConf.DEFAULT_QUEUE_NAME;
+ if (conf != null) {
+ queueName = conf.get(MRJobConfig.QUEUE_NAME, JobConf.DEFAULT_QUEUE_NAME);
+ }
+
MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer,
- user, jobName, jobId, forcedJobStateOnShutDown);
+ user, jobName, jobId, forcedJobStateOnShutDown, queueName);
fi.getJobSummary().setJobId(jobId);
fileMap.put(jobId, fi);
}
@@ -816,12 +822,14 @@ public class JobHistoryEventHandler exte
private String forcedJobStateOnShutDown;
MetaInfo(Path historyFile, Path conf, EventWriter writer, String user,
- String jobName, JobId jobId, String forcedJobStateOnShutDown) {
+ String jobName, JobId jobId, String forcedJobStateOnShutDown,
+ String queueName) {
this.historyFile = historyFile;
this.confFile = conf;
this.writer = writer;
this.jobIndexInfo =
- new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null);
+ new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null,
+ queueName);
this.jobSummary = new JobSummary();
this.flushTimer = new Timer("FlushTimer", true);
this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Fri Apr 18 16:32:35 2014
@@ -92,6 +92,10 @@ public class RMContainerAllocator extend
private static final Priority PRIORITY_REDUCE;
private static final Priority PRIORITY_MAP;
+ @VisibleForTesting
+ public static final String RAMPDOWN_DIAGNOSTIC = "Reducer preempted "
+ + "to make room for pending map attempts";
+
private Thread eventHandlingThread;
private final AtomicBoolean stopped;
@@ -1153,7 +1157,7 @@ public class RMContainerAllocator extend
TaskAttemptId id = reduceList.remove(0);//remove the one on top
LOG.info("Preempting " + id);
preemptionWaitingReduces.add(id);
- eventHandler.handle(new TaskAttemptEvent(id,
TaskAttemptEventType.TA_KILL));
+ eventHandler.handle(new TaskAttemptKillEvent(id, RAMPDOWN_DIAGNOSTIC));
}
}
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
Fri Apr 18 16:32:35 2014
@@ -362,6 +362,59 @@ public class TestRMContainerAllocator {
assigned, false);
}
+ @Test(timeout = 30000)
+ public void testReducerRampdownDiagnostics() throws Exception {
+ LOG.info("Running tesReducerRampdownDiagnostics");
+
+ final Configuration conf = new Configuration();
+ conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f);
+ final MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ final DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+ final RMApp app = rm.submitApp(1024);
+ dispatcher.await();
+
+ final String host = "host1";
+ final MockNM nm = rm.registerNode(String.format("%s:1234", host), 2048);
+ nm.nodeHeartbeat(true);
+ dispatcher.await();
+ final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ dispatcher.await();
+ final JobId jobId = MRBuilderUtils
+ .newJobId(appAttemptId.getApplicationId(), 0);
+ final Job mockJob = mock(Job.class);
+ when(mockJob.getReport()).thenReturn(
+ MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
+ final MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
+ appAttemptId, mockJob);
+ // add resources to scheduler
+ dispatcher.await();
+
+ // create the container request
+ final String[] locations = new String[] { host };
+ allocator.sendRequest(createReq(jobId, 0, 1024, locations, false, true));
+ for (int i = 0; i < 1;) {
+ dispatcher.await();
+ i += allocator.schedule().size();
+ nm.nodeHeartbeat(true);
+ }
+
+ allocator.sendRequest(createReq(jobId, 0, 1024, locations, true, false));
+ while (allocator.getTaskAttemptKillEvents().size() == 0) {
+ dispatcher.await();
+ allocator.schedule().size();
+ nm.nodeHeartbeat(true);
+ }
+ final String killEventMessage = allocator.getTaskAttemptKillEvents().get(0)
+ .getMessage();
+ Assert.assertTrue("No reducer rampDown preemption message",
+ killEventMessage.contains(RMContainerAllocator.RAMPDOWN_DIAGNOSTIC));
+ }
+
@Test
public void testMapReduceScheduling() throws Exception {
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
Fri Apr 18 16:32:35 2014
@@ -94,7 +94,7 @@ public class FileNameIndexUtils {
sb.append(DELIMITER);
//QueueName
- sb.append(escapeDelimiters(indexInfo.getQueueName()));
+ sb.append(escapeDelimiters(getQueueName(indexInfo)));
sb.append(DELIMITER);
//JobStartTime
@@ -262,6 +262,10 @@ public class FileNameIndexUtils {
return getNonEmptyString(indexInfo.getJobName());
}
+ private static String getQueueName(JobIndexInfo indexInfo) {
+ return getNonEmptyString(indexInfo.getQueueName());
+ }
+
//TODO Maybe handle default values for longs and integers here?
private static String getNonEmptyString(String in) {
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
Fri Apr 18 16:32:35 2014
@@ -18,10 +18,11 @@
package org.apache.hadoop.mapreduce.v2.jobhistory;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
/**
- * Maintains information which may be used by the jobHistroy indexing
+ * Maintains information which may be used by the jobHistory indexing
* system.
*/
public class JobIndexInfo {
@@ -41,6 +42,13 @@ public class JobIndexInfo {
public JobIndexInfo(long submitTime, long finishTime, String user,
String jobName, JobId jobId, int numMaps, int numReduces, String
jobStatus) {
+ this(submitTime, finishTime, user, jobName, jobId, numMaps, numReduces,
+ jobStatus, JobConf.DEFAULT_QUEUE_NAME);
+ }
+
+ public JobIndexInfo(long submitTime, long finishTime, String user,
+ String jobName, JobId jobId, int numMaps, int numReduces,
+ String jobStatus, String queueName) {
this.submitTime = submitTime;
this.finishTime = finishTime;
this.user = user;
@@ -50,8 +58,9 @@ public class JobIndexInfo {
this.numReduces = numReduces;
this.jobStatus = jobStatus;
this.jobStartTime = -1;
+ this.queueName = queueName;
}
-
+
public long getSubmitTime() {
return submitTime;
}
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Fri Apr 18 16:32:35 2014
@@ -747,7 +747,6 @@ public class JobConf extends Configurati
/**
* Should the map outputs be compressed before transfer?
- * Uses the SequenceFile compression.
*
* @param compress should the map outputs be compressed?
*/
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
Fri Apr 18 16:32:35 2014
@@ -27,8 +27,6 @@ import static org.apache.hadoop.mapreduc
@InterfaceAudience.Private
public class Limits {
- static final Configuration conf = new JobConf();
-
private int totalCounters;
private LimitExceededException firstViolation;
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
Fri Apr 18 16:32:35 2014
@@ -33,7 +33,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.record.Utils;
import com.google.common.base.Charsets;
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
Fri Apr 18 16:32:35 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.util.Shell;
@Private
@Unstable
@@ -65,8 +66,8 @@ public class HistoryServerFileSystemStat
private static final String TMP_FILE_PREFIX = "tmp-";
private static final FsPermission DIR_PERMISSIONS =
new FsPermission((short)0700);
- private static final FsPermission FILE_PERMISSIONS =
- new FsPermission((short)0400);
+ private static final FsPermission FILE_PERMISSIONS = Shell.WINDOWS
+ ? new FsPermission((short) 0700) : new FsPermission((short) 0400);
private static final int NUM_TOKEN_BUCKETS = 1000;
private FileSystem fs;
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
Fri Apr 18 16:32:35 2014
@@ -173,7 +173,7 @@ public class TestHSAdminServer {
Throwable th = null;
try {
- ProxyUsers.authorize(ugi, "127.0.0.1", conf);
+ ProxyUsers.authorize(ugi, "127.0.0.1");
} catch (Exception e) {
th = e;
}
@@ -189,7 +189,7 @@ public class TestHSAdminServer {
// resetting th
th = null;
try {
- ProxyUsers.authorize(ugi, "127.0.0.1", conf);
+ ProxyUsers.authorize(ugi, "127.0.0.1");
} catch (Exception e) {
th = e;
}
@@ -202,7 +202,7 @@ public class TestHSAdminServer {
th = null;
try {
- ProxyUsers.authorize(ugi, "127.0.0.1", conf);
+ ProxyUsers.authorize(ugi, "127.0.0.1");
} catch (Exception e) {
th = e;
}
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
Fri Apr 18 16:32:35 2014
@@ -182,7 +182,6 @@
<property><!--Loaded from
job.xml--><name>file.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from
job.xml--><name>dfs.namenode.fs-limits.max-directory-items</name><value>0</value></property>
<property><!--Loaded from
job.xml--><name>io.mapfile.bloom.size</name><value>1048576</value></property>
-<property><!--Loaded from
job.xml--><name>fs.hsftp.impl</name><value>org.apache.hadoop.hdfs.HsftpFileSystem</value></property>
<property><!--Loaded from
job.xml--><name>yarn.nodemanager.container-executor.class</name><value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value></property>
<property><!--Loaded from
job.xml--><name>mapreduce.map.maxattempts</name><value>4</value></property>
<property><!--Loaded from
job.xml--><name>mapreduce.jobtracker.jobhistory.block.size</name><value>3145728</value></property>
@@ -224,7 +223,6 @@
<property><!--Loaded from
job.xml--><name>hadoop.http.authentication.simple.anonymous.allowed</name><value>true</value></property>
<property><!--Loaded from
job.xml--><name>hadoop.rpc.socket.factory.class.default</name><value>org.apache.hadoop.net.StandardSocketFactory</value></property>
<property><!--Loaded from
job.xml--><name>mapreduce.job.submithostname</name><value>localhost</value></property>
-<property><!--Loaded from
job.xml--><name>fs.hftp.impl</name><value>org.apache.hadoop.hdfs.HftpFileSystem</value></property>
<property><!--Loaded from
job.xml--><name>dfs.namenode.handler.count</name><value>10</value></property>
<property><!--Loaded from
job.xml--><name>fs.automatic.close</name><value>true</value></property>
<property><!--Loaded from
job.xml--><name>fs.kfs.impl</name><value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value></property>
@@ -392,6 +390,7 @@
<property><!--Loaded from
job.xml--><name>mapreduce.job.counters.limit</name><value>120</value></property>
<property><!--Loaded from
job.xml--><name>dfs.datanode.ipc.address</name><value>0.0.0.0:50020</value></property>
<property><!--Loaded from
job.xml--><name>fs.webhdfs.impl</name><value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value></property>
+<property><!--Loaded from
job.xml--><name>fs.swebhdfs.impl</name><value>org.apache.hadoop.hdfs.web.SWebHdfsFileSystem</value></property>
<property><!--Loaded from
job.xml--><name>yarn.nodemanager.delete.debug-delay-sec</name><value>0</value></property>
<property><!--Loaded from
job.xml--><name>dfs.datanode.max.transfer.threads</name><value>4096</value></property>
</configuration>
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/SleepJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/SleepJob.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/SleepJob.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/SleepJob.java
Fri Apr 18 16:32:35 2014
@@ -219,7 +219,6 @@ public class SleepJob extends Configured
Job job = Job.getInstance(conf, "sleep");
job.setNumReduceTasks(numReducer);
job.setJarByClass(SleepJob.class);
- job.setNumReduceTasks(numReducer);
job.setMapperClass(SleepMapper.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(NullWritable.class);
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java
Fri Apr 18 16:32:35 2014
@@ -565,18 +565,6 @@ public class TestFileSystem extends Test
new Path("file:///").getFileSystem(conf);
FileSystem.closeAll();
}
-
- {
- Configuration conf = new Configuration();
- new Path("hftp://localhost:12345/";).getFileSystem(conf);
- FileSystem.closeAll();
- }
-
- {
- Configuration conf = new Configuration();
- FileSystem fs = new Path("hftp://localhost:12345/";).getFileSystem(conf);
- FileSystem.closeAll();
- }
}
public void testFsShutdownHook() throws Exception {
@@ -617,12 +605,12 @@ public class TestFileSystem extends Test
Configuration conf = new Configuration();
// check basic equality
- FileSystem.Cache.Key lowercaseCachekey1 = new FileSystem.Cache.Key(new
URI("hftp://localhost:12345/";), conf);
- FileSystem.Cache.Key lowercaseCachekey2 = new FileSystem.Cache.Key(new
URI("hftp://localhost:12345/";), conf);
+ FileSystem.Cache.Key lowercaseCachekey1 = new FileSystem.Cache.Key(new
URI("hdfs://localhost:12345/"), conf);
+ FileSystem.Cache.Key lowercaseCachekey2 = new FileSystem.Cache.Key(new
URI("hdfs://localhost:12345/"), conf);
assertEquals( lowercaseCachekey1, lowercaseCachekey2 );
// check insensitive equality
- FileSystem.Cache.Key uppercaseCachekey = new FileSystem.Cache.Key(new
URI("HFTP://Localhost:12345/"), conf);
+ FileSystem.Cache.Key uppercaseCachekey = new FileSystem.Cache.Key(new
URI("HDFS://Localhost:12345/"), conf);
assertEquals( lowercaseCachekey2, uppercaseCachekey );
// check behaviour with collections
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java
Fri Apr 18 16:32:35 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.pipes.TestPipeApplication.FakeSplit;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -69,7 +70,8 @@ public class TestPipesNonJavaInputFormat
}
// set data for splits
conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
- input1.getAbsolutePath() + "," + input2.getAbsolutePath());
+ StringUtils.escapeString(input1.getAbsolutePath()) + ","
+ + StringUtils.escapeString(input2.getAbsolutePath()));
InputSplit[] splits = inputFormat.getSplits(conf, 2);
assertEquals(2, splits.length);
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
Fri Apr 18 16:32:35 2014
@@ -250,7 +250,7 @@ public class RandomTextWriter extends Co
}
/**
- * A random list of 100 words from /usr/share/dict/words
+ * A random list of 1000 words from /usr/share/dict/words
*/
private static String[] words = {
"diurnalness", "Homoiousian",
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SleepJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SleepJob.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SleepJob.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SleepJob.java
Fri Apr 18 16:32:35 2014
@@ -208,7 +208,6 @@ public class SleepJob extends Configured
Job job = Job.getInstance(conf, "sleep");
job.setNumReduceTasks(numReducer);
job.setJarByClass(SleepJob.class);
- job.setNumReduceTasks(numReducer);
job.setMapperClass(SleepMapper.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(NullWritable.class);
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
Fri Apr 18 16:32:35 2014
@@ -24,6 +24,8 @@ import java.util.List;
import junit.framework.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -38,6 +40,9 @@ import org.junit.Test;
*/
public class TestMapReduceJobControl extends HadoopTestCase {
+ public static final Log LOG =
+ LogFactory.getLog(TestMapReduceJobControl.class.getName());
+
static Path rootDataDir = new Path(
System.getProperty("test.build.data", "."), "TestData");
static Path indir = new Path(rootDataDir, "indir");
@@ -117,6 +122,7 @@ public class TestMapReduceJobControl ext
}
public void testJobControlWithFailJob() throws Exception {
+ LOG.info("Starting testJobControlWithFailJob");
Configuration conf = createJobConf();
cleanupData(conf);
@@ -139,6 +145,8 @@ public class TestMapReduceJobControl ext
}
public void testJobControlWithKillJob() throws Exception {
+ LOG.info("Starting testJobControlWithKillJob");
+
Configuration conf = createJobConf();
cleanupData(conf);
Job job1 = MapReduceTestUtil.createKillJob(conf, outdir_1, indir);
@@ -175,6 +183,8 @@ public class TestMapReduceJobControl ext
}
public void testJobControl() throws Exception {
+ LOG.info("Starting testJobControl");
+
Configuration conf = createJobConf();
cleanupData(conf);
@@ -193,10 +203,12 @@ public class TestMapReduceJobControl ext
@Test(timeout = 30000)
public void testControlledJob() throws Exception {
+ LOG.info("Starting testControlledJob");
+
Configuration conf = createJobConf();
cleanupData(conf);
Job job1 = MapReduceTestUtil.createCopyJob(conf, outdir_1, indir);
- createDependencies(conf, job1);
+ JobControl theControl = createDependencies(conf, job1);
while (cjob1.getJobState() != ControlledJob.State.RUNNING) {
try {
Thread.sleep(100);
@@ -205,5 +217,10 @@ public class TestMapReduceJobControl ext
}
}
Assert.assertNotNull(cjob1.getMapredJobId());
+
+ // wait till all the jobs complete
+ waitTillAllFinished(theControl);
+ assertEquals("Some jobs failed", 0, theControl.getFailedJobList().size());
+ theControl.stop();
}
}
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Fri Apr 18 16:32:35 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.FailingMapper;
import org.apache.hadoop.RandomTextWriterJob;
import org.apache.hadoop.RandomTextWriterJob.RandomInputFormat;
import org.apache.hadoop.SleepJob;
+import org.apache.hadoop.SleepJob.SleepMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -98,6 +99,7 @@ public class TestMRJobs {
private static final EnumSet<RMAppState> TERMINAL_RM_APP_STATES =
EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);
private static final int NUM_NODE_MGRS = 3;
+ private static final String TEST_IO_SORT_MB = "11";
protected static MiniMRYarnCluster mrCluster;
protected static MiniDFSCluster dfsCluster;
@@ -205,6 +207,38 @@ public class TestMRJobs {
// JobStatus?)--compare against MRJobConfig.JOB_UBERTASK_ENABLE value
}
+ @Test(timeout = 300000)
+ public void testJobClassloader() throws IOException, InterruptedException,
+ ClassNotFoundException {
+ LOG.info("\n\n\nStarting testJobClassloader().");
+
+ if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+ final Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set master address to local to test that local mode applied iff
framework == local
+ sleepConf.set(MRConfig.MASTER_ADDRESS, "local");
+ sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
+ sleepConf.set(MRJobConfig.IO_SORT_MB, TEST_IO_SORT_MB);
+ sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
+ sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
+ sleepConf.set(MRJobConfig.REDUCE_LOG_LEVEL, Level.ALL.toString());
+ sleepConf.set(MRJobConfig.MAP_JAVA_OPTS, "-verbose:class");
+ final SleepJob sleepJob = new SleepJob();
+ sleepJob.setConf(sleepConf);
+ final Job job = sleepJob.createJob(1, 1, 10, 1, 10, 1);
+ job.setMapperClass(ConfVerificationMapper.class);
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.setJarByClass(SleepJob.class);
+ job.setMaxMapAttempts(1); // speed up failures
+ job.submit();
+ boolean succeeded = job.waitForCompletion(true);
+ Assert.assertTrue("Job status: " + job.getStatus().getFailureInfo(),
+ succeeded);
+ }
+
protected void verifySleepJobCounters(Job job) throws InterruptedException,
IOException {
Counters counters = job.getCounters();
@@ -795,4 +829,18 @@ public class TestMRJobs {
jos.closeEntry();
jarFile.delete();
}
+
+ public static class ConfVerificationMapper extends SleepMapper {
+ @Override
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ super.setup(context);
+ final Configuration conf = context.getConfiguration();
+ final String ioSortMb = conf.get(MRJobConfig.IO_SORT_MB);
+ if (!TEST_IO_SORT_MB.equals(ioSortMb)) {
+ throw new IOException("io.sort.mb expected: " + TEST_IO_SORT_MB
+ + ", actual: " + ioSortMb);
+ }
+ }
+ }
}
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
Fri Apr 18 16:32:35 2014
@@ -116,7 +116,7 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("HostName"));
Assert.assertNotNull(n.get("Rack"));
Assert.assertTrue("Node " + n.get("NodeId") + " should be RUNNING",
- n.get("State").getValueAsText().contains("RUNNING"));
+ n.get("State").asText().contains("RUNNING"));
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
@@ -124,10 +124,10 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
n.get("NodeId") + ": Unexpected number of used containers",
- 0, n.get("NumContainers").getValueAsInt());
+ 0, n.get("NumContainers").asInt());
Assert.assertEquals(
n.get("NodeId") + ": Unexpected amount of used memory",
- 0, n.get("UsedMemoryMB").getValueAsInt());
+ 0, n.get("UsedMemoryMB").asInt());
Assert.assertNotNull(n.get("AvailableMemoryMB"));
}
}
@@ -153,7 +153,7 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("HostName"));
Assert.assertNotNull(n.get("Rack"));
Assert.assertTrue("Node " + n.get("NodeId") + " should be RUNNING",
- n.get("State").getValueAsText().contains("RUNNING"));
+ n.get("State").asText().contains("RUNNING"));
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
---
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
(original)
+++
hadoop/common/branches/HADOOP-10388/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
Fri Apr 18 16:32:35 2014
@@ -250,7 +250,7 @@ public class RandomTextWriter extends Co
}
/**
- * A random list of 100 words from /usr/share/dict/words
+ * A random list of 1000 words from /usr/share/dict/words
*/
private static String[] words = {
"diurnalness", "Homoiousian",
