Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1556097&r1=1556096&r2=1556097&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Tue Jan 7 01:07:33 2014 @@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.event.Disp import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; +import com.google.common.annotations.VisibleForTesting; + /****************************************************************** * {@link JobHistoryServer} is responsible for servicing all job history * related requests from client. @@ -60,10 +62,10 @@ public class JobHistoryServer extends Co public static final long historyServerTimeStamp = System.currentTimeMillis(); private static final Log LOG = LogFactory.getLog(JobHistoryServer.class); - private HistoryContext historyContext; + protected HistoryContext historyContext; private HistoryClientService clientService; private JobHistory jobHistoryService; - private JHSDelegationTokenSecretManager jhsDTSecretManager; + protected JHSDelegationTokenSecretManager jhsDTSecretManager; private AggregatedLogDeletionService aggLogDelService; private HSAdminServer hsAdminServer; private HistoryServerStateStoreService stateStore; @@ -129,8 +131,7 @@ public class JobHistoryServer extends Co historyContext = (HistoryContext)jobHistoryService; stateStore = createStateStore(conf); this.jhsDTSecretManager = createJHSSecretManager(conf, stateStore); - clientService = new HistoryClientService(historyContext, - this.jhsDTSecretManager); + clientService = createHistoryClientService(); aggLogDelService = new AggregatedLogDeletionService(); hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService); addService(stateStore); @@ -142,6 +143,12 @@ public class JobHistoryServer extends Co super.serviceInit(config); } + @VisibleForTesting + protected HistoryClientService createHistoryClientService() { + return new HistoryClientService(historyContext, + this.jhsDTSecretManager); + } + protected JHSDelegationTokenSecretManager createJHSSecretManager( Configuration conf, HistoryServerStateStoreService store) { long secretKeyInterval =
Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java?rev=1556097&r1=1556096&r2=1556097&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java Tue Jan 7 01:07:33 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.mapred; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; -import java.util.Arrays; import java.util.List; import junit.framework.TestCase; @@ -29,20 +28,17 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.ipc.ProtocolSignature; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.OutputFormat; -import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent; +import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.split.JobSplitWriter; -import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader; -import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; +import org.apache.hadoop.mapreduce.split.JobSplitWriter; +import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader; import org.apache.hadoop.util.ReflectionUtils; /** @@ -110,11 +106,16 @@ public class TestMapProgress extends Tes statusUpdate(taskId, taskStatus); } + public void preempted(TaskAttemptID taskId, TaskStatus taskStatus) + throws IOException, InterruptedException { + statusUpdate(taskId, taskStatus); + } + public boolean canCommit(TaskAttemptID taskid) throws IOException { return true; } - public boolean statusUpdate(TaskAttemptID taskId, TaskStatus taskStatus) + public AMFeedback statusUpdate(TaskAttemptID taskId, TaskStatus taskStatus) throws IOException, InterruptedException { StringBuffer buf = new StringBuffer("Task "); buf.append(taskId); @@ -128,7 +129,9 @@ public class TestMapProgress extends Tes LOG.info(buf.toString()); // ignore phase // ignore counters - return true; + AMFeedback a = new AMFeedback(); + a.setTaskFound(true); + return a; } public void reportDiagnosticInfo(TaskAttemptID taskid, String trace) throws IOException { @@ -145,6 +148,17 @@ public class TestMapProgress extends Tes SortedRanges.Range range) throws IOException { LOG.info("Task " + taskid + " reportedNextRecordRange " + range); } + + @Override + public TaskCheckpointID getCheckpointID(TaskID taskId) { + // do nothing + return null; + } + + @Override + public void setCheckpointID(TaskID downgrade, TaskCheckpointID cid) { + // do nothing + } } private FileSystem fs = null; Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java?rev=1556097&r1=1556096&r2=1556097&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java Tue Jan 7 01:07:33 2014 @@ -27,6 +27,10 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ProtocolSignature; import org.apache.hadoop.mapred.SortedRanges.Range; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.checkpoint.CheckpointID; +import org.apache.hadoop.mapreduce.checkpoint.FSCheckpointID; +import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID; + public class TestTaskCommit extends HadoopTestCase { Path rootDir = @@ -132,11 +136,6 @@ public class TestTaskCommit extends Hado } @Override - public boolean ping(TaskAttemptID taskid) throws IOException { - return true; - } - - @Override public void reportDiagnosticInfo(TaskAttemptID taskid, String trace) throws IOException { } @@ -152,9 +151,11 @@ public class TestTaskCommit extends Hado } @Override - public boolean statusUpdate(TaskAttemptID taskId, TaskStatus taskStatus) + public AMFeedback statusUpdate(TaskAttemptID taskId, TaskStatus taskStatus) throws IOException, InterruptedException { - return true; + AMFeedback a = new AMFeedback(); + a.setTaskFound(true); + return a; } @Override @@ -168,6 +169,22 @@ public class TestTaskCommit extends Hado long clientVersion, int clientMethodsHash) throws IOException { return null; } + + @Override + public void preempted(TaskAttemptID taskId, TaskStatus taskStatus) + throws IOException, InterruptedException { + fail("Task should not go to commit-pending"); + } + + @Override + public TaskCheckpointID getCheckpointID(TaskID taskId) { + return null; + } + + @Override + public void setCheckpointID(TaskID downgrade, TaskCheckpointID cid) { + // ignore + } } /** Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java?rev=1556097&r1=1556096&r2=1556097&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java Tue Jan 7 01:07:33 2014 @@ -39,6 +39,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.hs.HistoryClientService; import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService; import org.apache.hadoop.mapreduce.v2.hs.JHSDelegationTokenSecretManager; import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; @@ -94,6 +95,17 @@ public class TestJHSSecurity { return new JHSDelegationTokenSecretManager(initialInterval, maxLifetime, renewInterval, 3600000, store); } + + @Override + protected HistoryClientService createHistoryClientService() { + return new HistoryClientService(historyContext, + this.jhsDTSecretManager) { + @Override + protected void initializeWebApp(Configuration conf) { + // Don't need it, skip.; + } + }; + } }; // final JobHistoryServer jobHistoryServer = jhServer; jobHistoryServer.init(conf); Modified: hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestUmbilicalProtocolWithJobToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestUmbilicalProtocolWithJobToken.java?rev=1556097&r1=1556096&r2=1556097&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestUmbilicalProtocolWithJobToken.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestUmbilicalProtocolWithJobToken.java Tue Jan 7 01:07:33 2014 @@ -115,7 +115,7 @@ public class TestUmbilicalProtocolWithJo proxy = (TaskUmbilicalProtocol) RPC.getProxy( TaskUmbilicalProtocol.class, TaskUmbilicalProtocol.versionID, addr, conf); - proxy.ping(null); + proxy.statusUpdate(null, null); } finally { server.stop(); if (proxy != null) {
