Author: todd
Date: Wed Apr 4 20:00:15 2012
New Revision: 1309576
URL: http://svn.apache.org/viewvc?rev=1309576&view=rev
Log:
Merge trunk into auto-HA branch
Resolved some trivial conflicts in NNHAServiceTarget
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/c++/ (props
changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/block_forensics/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build-contrib.xml
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build.xml
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/data_join/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/
(props changed)
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1309162-1309567
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt Wed
Apr 4 20:00:15 2012
@@ -139,6 +139,8 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-3955. Change MR to use ProtobufRpcEngine from hadoop-common
instead of ProtoOverHadoopRpcEngine. (Jitendra Nath Pandey via sseth)
+ MAPREDUCE-4103. Fix HA docs for changes to shell command fencer args (todd)
+
OPTIMIZATIONS
BUG FIXES
@@ -197,7 +199,9 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4095. TestJobInProgress#testLocality uses a bogus topology.
(Colin Patrick McCabe via eli)
-
+
+ MAPREDUCE-4098. TestMRApps testSetClasspath fails (tucu)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -240,6 +244,16 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4020. Web services returns incorrect JSON for deep queue tree
(Anupam Seth via tgraves)
+ MAPREDUCE-3672. Killed maps shouldn't be counted towards
+ JobCounter.NUM_FAILED_MAPS. (Anupam Seth via tgraves)
+
+ MAPREDUCE-3682 Tracker URL says AM tasks run on localhost.
+ (Ravi Prakash via tgraves)
+
+ MAPREDUCE-3082. Archive command take wrong path for input file with current
+ directory (John George via bobby)
+
+ MAPREDUCE-3650. testGetTokensForHftpFS() fails (Ravi Prakash via bobby)
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1309162-1309567
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1309162-1309567
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
Wed Apr 4 20:00:15 2012
@@ -643,7 +643,8 @@ public class MRAppMaster extends Composi
public synchronized void start() {
if (job.isUber()) {
this.containerAllocator = new LocalContainerAllocator(
- this.clientService, this.context);
+ this.clientService, this.context, nmHost, nmPort, nmHttpPort
+ , containerID);
} else {
this.containerAllocator = new RMContainerAllocator(
this.clientService, this.context);
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Apr 4 20:00:15 2012
@@ -988,6 +988,23 @@ public abstract class TaskAttemptImpl im
}
return jce;
}
+
+ private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
+ TaskAttemptImpl taskAttempt) {
+ TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
+ JobCounterUpdateEvent jce = new
JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
+
+ long slotMillisIncrement = computeSlotMillis(taskAttempt);
+
+ if (taskType == TaskType.MAP) {
+ jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
+ jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
+ } else {
+ jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
+ jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES,
slotMillisIncrement);
+ }
+ return jce;
+ }
private static
TaskAttemptUnsuccessfulCompletionEvent
@@ -1214,8 +1231,13 @@ public abstract class TaskAttemptImpl im
TaskAttemptUnsuccessfulCompletionEvent tauce =
createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
finalState);
- taskAttempt.eventHandler
+ if(finalState == TaskAttemptState.FAILED) {
+ taskAttempt.eventHandler
.handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+ } else if(finalState == TaskAttemptState.KILLED) {
+ taskAttempt.eventHandler
+ .handle(createJobCounterUpdateEventTAKilled(taskAttempt));
+ }
taskAttempt.eventHandler.handle(new JobHistoryEvent(
taskAttempt.attemptId.getTaskId().getJobId(), tauce));
} else {
@@ -1441,7 +1463,7 @@ public abstract class TaskAttemptImpl im
taskAttempt.setFinishTime();
if (taskAttempt.getLaunchTime() != 0) {
taskAttempt.eventHandler
- .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+ .handle(createJobCounterUpdateEventTAKilled(taskAttempt));
TaskAttemptUnsuccessfulCompletionEvent tauce =
createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
TaskAttemptState.KILLED);
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
Wed Apr 4 20:00:15 2012
@@ -65,14 +65,23 @@ public class LocalContainerAllocator ext
private AtomicInteger containerCount = new AtomicInteger();
private long retryInterval;
private long retrystartTime;
+ private String nmHost;
+ private int nmPort;
+ private int nmHttpPort;
+ private ContainerId containerId;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
public LocalContainerAllocator(ClientService clientService,
- AppContext context) {
+ AppContext context, String nmHost, int nmPort, int nmHttpPort
+ , ContainerId cId) {
super(clientService, context);
this.eventHandler = context.getEventHandler();
+ this.nmHost = nmHost;
+ this.nmPort = nmPort;
+ this.nmHttpPort = nmHttpPort;
+ this.containerId = cId;
}
@Override
@@ -131,17 +140,17 @@ public class LocalContainerAllocator ext
LOG.info("Processing the event " + event.toString());
ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
cID.setApplicationAttemptId(applicationAttemptId);
- // use negative ids to denote that these are local. Need a better way ??
- cID.setId((-1) * containerCount.getAndIncrement());
+ // Assign the same container ID as the AM
+ cID.setId(this.containerId.getId());
Container container = recordFactory.newRecordInstance(Container.class);
container.setId(cID);
NodeId nodeId = Records.newRecord(NodeId.class);
- nodeId.setHost("localhost");
- nodeId.setPort(1234);
+ nodeId.setHost(this.nmHost);
+ nodeId.setPort(this.nmPort);
container.setNodeId(nodeId);
container.setContainerToken(null);
- container.setNodeHttpAddress("localhost:8042");
+ container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
// send the container-assigned event to task attempt
if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Wed Apr 4 20:00:15 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.junit.Test;
@@ -130,15 +131,12 @@ public class TestMRApps {
Job job = Job.getInstance();
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration());
- assertEquals("$PWD:$HADOOP_CONF_DIR:" +
- "$HADOOP_COMMON_HOME/share/hadoop/common/*:" +
- "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*:" +
- "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*:" +
- "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*:" +
- "$YARN_HOME/share/hadoop/mapreduce/*:" +
- "$YARN_HOME/share/hadoop/mapreduce/lib/*:" +
- "job.jar:$PWD/*",
- environment.get("CLASSPATH"));
+ assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+ String confClasspath =
job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
+ if (confClasspath != null) {
+ confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
+ }
+ assertTrue(environment.get("CLASSPATH").contains(confClasspath));
}
@Test public void testSetClasspathWithUserPrecendence() {
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
Wed Apr 4 20:00:15 2012
@@ -27,6 +27,8 @@ import org.apache.hadoop.classification.
public enum JobCounter {
NUM_FAILED_MAPS,
NUM_FAILED_REDUCES,
+ NUM_KILLED_MAPS,
+ NUM_KILLED_REDUCES,
TOTAL_LAUNCHED_MAPS,
TOTAL_LAUNCHED_REDUCES,
OTHER_LOCAL_MAPS,
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1309162-1309567
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
Wed Apr 4 20:00:15 2012
@@ -16,6 +16,8 @@ CounterGroupName= Job C
NUM_FAILED_MAPS.name= Failed map tasks
NUM_FAILED_REDUCES.name= Failed reduce tasks
+NUM_KILLED_MAPS.name= Killed map tasks
+NUM_KILLED_REDUCES.name= Killed reduce tasks
TOTAL_LAUNCHED_MAPS.name= Launched map tasks
TOTAL_LAUNCHED_REDUCES.name= Launched reduce tasks
OTHER_LOCAL_MAPS.name= Other local map tasks
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
Wed Apr 4 20:00:15 2012
@@ -195,6 +195,8 @@ public class TestJobCleanup {
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
job.waitForCompletion();
+ Counters counters = job.getCounters();
+ assertTrue("No. of failed maps should be
1",counters.getCounter(JobCounter.NUM_FAILED_MAPS) == 1);
if (fileName != null) {
Path testFile = new Path(outDir, fileName);
@@ -240,6 +242,9 @@ public class TestJobCleanup {
job.killJob(); // kill the job
job.waitForCompletion(); // wait for the job to complete
+
+ counters = job.getCounters();
+ assertTrue("No. of killed maps should be 1",
counters.getCounter(JobCounter.NUM_KILLED_MAPS) == 1);
if (fileName != null) {
Path testFile = new Path(outDir, fileName);
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
Wed Apr 4 20:00:15 2012
@@ -234,8 +234,10 @@ public class TestSpeculativeExecution {
.getValue());
Assert.assertEquals(2,
counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES)
.getValue());
- Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
+ Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
.getValue());
+ Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_KILLED_MAPS)
+ .getValue());
/*----------------------------------------------------------------------
* Test that Reducer speculates if REDUCE_SPECULATIVE is true and
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
Wed Apr 4 20:00:15 2012
@@ -322,13 +322,40 @@ HDFS High Availability
The string between '(' and ')' is passed directly to a bash shell and
may not
include any closing parentheses.
- When executed, the first argument to the configured script will be the
address
- of the NameNode to be fenced, followed by all arguments specified in the
- configuration.
-
The shell command will be run with an environment set up to contain all
of the
current Hadoop configuration variables, with the '_' character replacing
any
- '.' characters in the configuration keys. If the shell command returns
an exit
+ '.' characters in the configuration keys. The configuration used has
already had
+ any namenode-specific configurations promoted to their generic forms --
for example
+ <<dfs_namenode_rpc-address>> will contain the RPC address of the target
node, even
+ though the configuration may specify that variable as
+ <<dfs.namenode.rpc-address.ns1.nn1>>.
+
+ Additionally, the following variables referring to the target node to be
fenced
+ are also available:
+
+*-----------------------:-----------------------------------+
+| $target_host | hostname of the node to be fenced |
+*-----------------------:-----------------------------------+
+| $target_port | IPC port of the node to be fenced |
+*-----------------------:-----------------------------------+
+| $target_address | the above two, combined as host:port |
+*-----------------------:-----------------------------------+
+| $target_nameserviceid | the nameservice ID of the NN to be fenced |
+*-----------------------:-----------------------------------+
+| $target_namenodeid | the namenode ID of the NN to be fenced |
+*-----------------------:-----------------------------------+
+
+ These environment variables may also be used as substitutions in the
shell
+ command itself. For example:
+
+---
+<property>
+ <name>dfs.ha.fencing.methods</name>
+ <value>shell(/path/to/my/script.sh --nameservice=$target_nameserviceid
$target_host:$target_port)</value>
+</property>
+---
+
+ If the shell command returns an exit
code of 0, the fencing is determined to be successful. If it returns any
other
exit code, the fencing was not successful and the next fencing method in
the
list will be attempted.
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1309162-1309567
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1309162-1309567
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1309162-1309567
Modified:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
Wed Apr 4 20:00:15 2012
@@ -321,7 +321,7 @@ public class TestTokenCache {
final Token<DelegationTokenIdentifier> t =
new Token<DelegationTokenIdentifier>(dtId, dtSecretManager);
- final URI uri = new URI("hftp://host:2222/file1";);
+ final URI uri = new URI("hftp://127.0.0.1:2222/file1";);
final String fs_addr =
SecurityUtil.buildDTServiceName(uri, NameNode.DEFAULT_PORT);
t.setService(new Text(fs_addr));
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1309162-1309567
