Author: stevel
Date: Thu Jun 11 10:07:45 2009
New Revision: 783716
URL: http://svn.apache.org/viewvc?rev=783716&view=rev
Log:
SFOS-3628 integrate trunk (used for own hadoop-0.21-alpha-12 release)
Added:
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/
- copied from r783712, hadoop/core/trunk/src/contrib/dynamic-scheduler/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/README
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/README
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/build.xml
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/build.xml
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/ivy/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/ivy/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/ivy.xml
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/ivy.xml
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/ivy/libraries.properties
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/ivy/libraries.properties
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/AllocationStore.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/AllocationStore.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/BudgetQueue.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/BudgetQueue.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/DynamicPriorityScheduler.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/DynamicPriorityScheduler.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/DynamicPriorityServlet.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/DynamicPriorityServlet.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/FileAllocationStore.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/FileAllocationStore.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PriorityAuthorization.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PriorityAuthorization.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PriorityScheduler.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PriorityScheduler.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PrioritySchedulerOptions.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PrioritySchedulerOptions.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueAllocation.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueAllocation.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueAllocator.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueAllocator.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueTaskScheduler.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueTaskScheduler.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/
- copied from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/BaseSchedulerTest.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/BaseSchedulerTest.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/FakeDynamicScheduler.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/FakeDynamicScheduler.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/TestDynamicScheduler.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/TestDynamicScheduler.java
hadoop/core/branches/HADOOP-3628-2/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/TestPriorityScheduler.java
- copied unchanged from r783712,
hadoop/core/trunk/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/TestPriorityScheduler.java
Modified:
hadoop/core/branches/HADOOP-3628-2/ (props changed)
hadoop/core/branches/HADOOP-3628-2/CHANGES.txt (contents, props changed)
hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml
hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/ivy.xml
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java
Propchange: hadoop/core/branches/HADOOP-3628-2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 11 10:07:45 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:779103-783032
+/hadoop/core/trunk:779103-783712
Modified: hadoop/core/branches/HADOOP-3628-2/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/CHANGES.txt?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/CHANGES.txt (original)
+++ hadoop/core/branches/HADOOP-3628-2/CHANGES.txt Thu Jun 11 10:07:45 2009
@@ -798,6 +798,11 @@
HADOOP-5963. Remove an unnecessary exception catch in NNBench. (Boris
Shkolnik via szetszwo)
+ HADOOP-5989. Fix streaming test failure. (gkesavan)
+
+ HADOOP-5981. Fix a bug in HADOOP-2838 in parsing mapred.child.env.
+ (Amar Kamat via sharad)
+
Release 0.20.1 - Unreleased
INCOMPATIBLE CHANGES
@@ -916,6 +921,10 @@
HADOOP-5884. Fixes accounting in capacity scheduler so that high RAM jobs
take more slots. (Vinod Kumar Vavilapalli via yhemanth)
+ HADOOP-5869. Fix bug in assignment of setup / cleanup task that was
+ causing TestQueueCapacities to fail.
+ (Sreekanth Ramakrishnan via yhemanth)
+
Release 0.20.0 - 2009-04-15
INCOMPATIBLE CHANGES
Propchange: hadoop/core/branches/HADOOP-3628-2/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 11 10:07:45 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:779103-783032
+/hadoop/core/trunk/CHANGES.txt:779103-783712
Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml Thu Jun 11
10:07:45 2009
@@ -55,6 +55,7 @@
<fileset dir="." includes="fairscheduler/build.xml"/>
<fileset dir="." includes="capacity-scheduler/build.xml"/>
<fileset dir="." includes="mrunit/build.xml"/>
+ <fileset dir="." includes="dynamic-scheduler/build.xml"/>
</subant>
<available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
<fail if="testsfailed">Tests failed!</fail>
Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/ivy.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/ivy.xml?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/ivy.xml (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/ivy.xml Thu Jun 11
10:07:45 2009
@@ -28,6 +28,10 @@
name="commons-cli"
rev="${commons-cli2.version}"
conf="common->default"/>
+ <dependency org="commons-cli"
+ name="commons-cli"
+ rev="${commons-cli.version}"
+ conf="common->default"/>
<dependency org="commons-logging"
name="commons-logging"
rev="${commons-logging.version}"
Modified:
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
---
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
(original)
+++
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
Thu Jun 11 10:07:45 2009
@@ -919,18 +919,19 @@
boolean taskCleanup) {
// create the task
Task t = null;
- if (isMapTask() && !jobSetup && !jobCleanup) {
+ if (isMapTask()) {
LOG.debug("attempt " + numTaskFailures + " sending skippedRecords "
+ failedRanges.getIndicesCount());
-
- t =
- new MapTask(jobFile, taskid, partition, rawSplit.getClassName(),
- rawSplit.getBytes());
-
- } else if (jobSetup || jobCleanup) {
- t = new MapTask(jobFile, taskid, partition, null, new BytesWritable());
- }
- else {
+ String splitClass = null;
+ BytesWritable split;
+ if (!jobSetup && !jobCleanup) {
+ splitClass = rawSplit.getClassName();
+ split = rawSplit.getBytes();
+ } else {
+ split = new BytesWritable();
+ }
+ t = new MapTask(jobFile, taskid, partition, splitClass, split);
+ } else {
t = new ReduceTask(jobFile, taskid, partition, numMaps);
}
if (jobCleanup) {
Modified:
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
---
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++
hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
Thu Jun 11 10:07:45 2009
@@ -101,6 +101,7 @@
@Override
public final void run() {
+ String errorInfo = "Child Error";
try {
//before preparing the job localize
@@ -405,16 +406,36 @@
if (mapredChildEnv != null && mapredChildEnv.length() > 0) {
String childEnvs[] = mapredChildEnv.split(",");
for (String cEnv : childEnvs) {
- String[] parts = cEnv.split("="); // split on '='
- String value = env.get(parts[0]);
- if (value != null) {
- // replace $env with the tt's value of env
- value = parts[1].replaceAll("$" + parts[0], value);
- } else {
- // for cases where x=$x:/tmp is passed and x doesnt exist
- value = parts[1].replaceAll("$" + parts[0], "");
+ try {
+ String[] parts = cEnv.split("="); // split on '='
+ String value = env.get(parts[0]);
+ if (value != null) {
+ // replace $env with the child's env constructed by tt's
+ // example LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp
+ value = parts[1].replace("$" + parts[0], value);
+ } else {
+ // this key is not configured by the tt for the child .. get it
+ // from the tt's env
+ // example PATH=$PATH:/tmp
+ value = System.getenv(parts[0]);
+ if (value != null) {
+ // the env key is present in the tt's env
+ value = parts[1].replace("$" + parts[0], value);
+ } else {
+ // the env key is note present anywhere .. simply set it
+ // example X=$X:/tmp or X=/tmp
+ value = parts[1].replace("$" + parts[0], "");
+ }
+ }
+ env.put(parts[0], value);
+ } catch (Throwable t) {
+ // set the error msg
+ errorInfo = "Invalid User environment settings : " +
mapredChildEnv
+ + ". Failed to parse user-passed environment param."
+ + " Expecting : env1=value1,env2=value2...";
+ LOG.warn(errorInfo);
+ throw t;
}
- env.put(parts[0], value);
}
}
@@ -444,9 +465,10 @@
LOG.fatal(t.getTaskID()+" reporting FSError", ie);
}
} catch (Throwable throwable) {
- LOG.warn(t.getTaskID()+" Child Error", throwable);
+ LOG.warn(t.getTaskID() + errorInfo, throwable);
+ Throwable causeThrowable = new Throwable(errorInfo, throwable);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- throwable.printStackTrace(new PrintStream(baos));
+ causeThrowable.printStackTrace(new PrintStream(baos));
try {
tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString());
} catch (IOException e) {
Modified:
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
---
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java
(original)
+++
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java
Thu Jun 11 10:07:45 2009
@@ -416,6 +416,8 @@
conf.setInputFormat(ControlledMapReduceJob.class);
FileInputFormat.addInputPath(conf, new Path("ignored"));
conf.setOutputFormat(NullOutputFormat.class);
+ conf.setMapSpeculativeExecution(false);
+ conf.setReduceSpeculativeExecution(false);
// Set the following for reduce tasks to be able to be started running
// immediately along with maps.
@@ -573,4 +575,4 @@
return new ControlledMapReduceJobRunner(conf, numMappers, numReducers);
}
}
-}
\ No newline at end of file
+}
Modified:
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java?rev=783716&r1=783715&r2=783716&view=diff
==============================================================================
---
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java
(original)
+++
hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java
Thu Jun 11 10:07:45 2009
@@ -84,7 +84,8 @@
}
// configure a job
- private void configure(JobConf conf, Path inDir, Path outDir, String input)
+ private void configure(JobConf conf, Path inDir, Path outDir, String input,
+ Class<? extends Mapper> map)
throws IOException {
// set up the input file system and write input text.
FileSystem inFs = inDir.getFileSystem(conf);
@@ -102,7 +103,7 @@
// configure the mapred Job which creates a tempfile in map.
conf.setJobName("testmap");
- conf.setMapperClass(MapClass.class);
+ conf.setMapperClass(map);
conf.setReducerClass(IdentityReducer.class);
conf.setNumMapTasks(1);
conf.setNumReduceTasks(0);
@@ -126,7 +127,7 @@
Path outDir,
String input)
throws IOException {
- configure(conf, inDir, outDir, input);
+ configure(conf, inDir, outDir, input, MapClass.class);
FileSystem outFs = outDir.getFileSystem(conf);
@@ -149,17 +150,27 @@
// Mappers that simply checks if the desired user env are present or not
static class EnvCheckMapper extends MapReduceBase implements
Mapper<WritableComparable, Writable, WritableComparable, Writable> {
+ private static String PATH;
+
public void map(WritableComparable key, Writable value,
OutputCollector<WritableComparable, Writable> out, Reporter reporter)
throws IOException {
- // check if X=$X:/abc works
+ // check if the pwd is there in LD_LIBRARY_PATH
+ String pwd = System.getenv("PWD");
+ assertTrue("LD doesnt contain pwd",
+ System.getenv("LD_LIBRARY_PATH").contains(pwd));
+
+ // check if X=$X:/abc works for LD_LIBRARY_PATH
checkEnv("LD_LIBRARY_PATH", "/tmp", "append");
// check if X=/tmp works for an already existing parameter
checkEnv("HOME", "/tmp", "noappend");
- // check if my_path=/tmp for a new env variable
+ // check if X=/tmp for a new env variable
checkEnv("MY_PATH", "/tmp", "noappend");
- // check if new_path=$new_path:/tmp works and results into :/tmp
+ // check if X=$X:/tmp works for a new env var and results into :/tmp
checkEnv("NEW_PATH", ":/tmp", "noappend");
+ // check if X=$(tt's X var):/tmp for an old env variable inherited from
+ // the tt
+ checkEnv("PATH", PATH + ":/tmp", "noappend");
}
private void checkEnv(String envName, String expValue, String mode)
@@ -181,6 +192,10 @@
}
}
}
+
+ public void configure(JobConf conf) {
+ PATH = conf.get("path");
+ }
}
@Override
@@ -256,13 +271,20 @@
Path outDir = new Path("testing/wc/output1");
String input = "The input";
- configure(conf, inDir, outDir, input);
+ configure(conf, inDir, outDir, input, EnvCheckMapper.class);
FileSystem outFs = outDir.getFileSystem(conf);
+ // test
+ // - new SET of new var (MY_PATH)
+ // - set of old var (HOME)
+ // - append to an old var from modified env (LD_LIBRARY_PATH)
+ // - append to an old var from tt's env (PATH)
+ // - append to a new var (NEW_PATH)
conf.set("mapred.child.env",
"MY_PATH=/tmp,HOME=/tmp,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
- + "NEW_PATH=$NEW_PATH:/tmp");
+ + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
+ conf.set("path", System.getenv("PATH"));
JobClient.runJob(conf);
outFs.delete(outDir, true);
