Author: omalley
Date: Fri Mar 4 03:40:25 2011
New Revision: 1077099
URL: http://svn.apache.org/viewvc?rev=1077099&view=rev
Log:
commit 32ab54df050f8de4dee11e9b9c125a67a624f971
Author: Hemanth Yamijala <[email protected]>
Date: Mon Jan 11 19:38:08 2010 +0530
Reverting patch
https://issues.apache.org/jira/secure/attachment/12426091/hadoop-5771-ydist.patch
for HADOOP-5771
+++ b/YAHOO-CHANGES.txt
+ HADOOP-5771. Implements unit tests for LinuxTaskController.
+ (Sreekanth Ramakrishnan and Vinod Kumar Vavilapalli via yhemanth)
+
Removed:
hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingAsDifferentUser.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobExecutionAsDifferentUser.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipesAsDifferentUser.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/build.xml
hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
Modified: hadoop/common/branches/branch-0.20-security-patches/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/build.xml?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/build.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/build.xml Fri Mar 4
03:40:25 2011
@@ -740,8 +740,6 @@
<sysproperty key="test.debug.data" value="${test.debug.data}"/>
<sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
<sysproperty key="test.src.dir" value="${test.src.dir}"/>
- <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
- <sysproperty key="taskcontroller-user" value="${taskcontroller-user}"/>
<sysproperty key="test.build.extraconf" value="${test.build.extraconf}"
/>
<sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
<sysproperty key="java.library.path"
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
Fri Mar 4 03:40:25 2011
@@ -233,8 +233,6 @@
<sysproperty key="fs.default.name" value="${fs.default.name}"/>
<sysproperty key="hadoop.test.localoutputfile"
value="${hadoop.test.localoutputfile}"/>
<sysproperty key="hadoop.log.dir" value="${hadoop.log.dir}"/>
- <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
- <sysproperty key="taskcontroller-user" value="${taskcontroller-user}"/>
<classpath refid="test.classpath"/>
<formatter type="${test.junit.output.format}" />
<batchtest todir="${build.test}" unless="testcase">
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
Fri Mar 4 03:40:25 2011
@@ -41,6 +41,8 @@ public class TrApp
// test that some JobConf properties are exposed as expected
// Note the dots translated to underscore:
// property names have been escaped in PipeMapRed.safeEnvVarName()
+ expect("mapred_job_tracker", "local");
+ //expect("mapred_local_dir", "build/test/mapred/local");
expectDefined("mapred_local_dir");
expect("mapred_output_format_class",
"org.apache.hadoop.mapred.TextOutputFormat");
expect("mapred_output_key_class", "org.apache.hadoop.io.Text");
@@ -50,6 +52,7 @@ public class TrApp
expectDefined("mapred_task_id");
expectDefined("map_input_file");
+ expect("map_input_start", "0");
expectDefined("map_input_length");
expectDefined("io_sort_factor");
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
Fri Mar 4 03:40:25 2011
@@ -408,7 +408,7 @@ class LinuxTaskController extends TaskCo
File workDir, Map<String, String> env)
throws IOException {
String[] taskControllerCmd = new String[3 + cmdArgs.size()];
- taskControllerCmd[0] = getTaskControllerExecutablePath();
+ taskControllerCmd[0] = taskControllerExe;
taskControllerCmd[1] = userName;
taskControllerCmd[2] = String.valueOf(command.ordinal());
int i = 3;
@@ -475,10 +475,7 @@ class LinuxTaskController extends TaskCo
}
}
}
-
- protected String getTaskControllerExecutablePath() {
- return taskControllerExe;
- }
+
/**
* Sets up the permissions of the following directories:
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
Fri Mar 4 03:40:25 2011
@@ -415,23 +415,4 @@ public class TestMiniMRChildTask extends
RunningJob job = JobClient.runJob(conf);
assertTrue("The environment checker job failed.", job.isSuccessful());
}
-
- void runTestTaskEnv(JobConf conf, Path inDir, Path outDir) throws
IOException {
- String input = "The input";
- configure(conf, inDir, outDir, input, EnvCheckMapper.class,
- IdentityReducer.class);
- // test
- // - new SET of new var (MY_PATH)
- // - set of old var (HOME)
- // - append to an old var from modified env (LD_LIBRARY_PATH)
- // - append to an old var from tt's env (PATH)
- // - append to a new var (NEW_PATH)
- conf.set("mapred.child.env",
- "MY_PATH=/tmp,HOME=/tmp,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
- + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
- conf.set("path", System.getenv("PATH"));
- RunningJob job = JobClient.runJob(conf);
- assertTrue("The environment checker job failed.", job.isSuccessful());
- }
-
}
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
Fri Mar 4 03:40:25 2011
@@ -32,8 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
@@ -49,18 +47,7 @@ import org.apache.hadoop.util.ToolRunner
public class TestPipes extends TestCase {
private static final Log LOG =
LogFactory.getLog(TestPipes.class.getName());
-
- private static Path cppExamples =
- new Path(System.getProperty("install.c++.examples"));
- static Path wordCountSimple =
- new Path(cppExamples, "bin/wordcount-simple");
- static Path wordCountPart =
- new Path(cppExamples, "bin/wordcount-part");
- static Path wordCountNoPipes =
- new Path(cppExamples,"bin/wordcount-nopipe");
-
- static Path nonPipedOutDir;
-
+
static void cleanup(FileSystem fs, Path p) throws IOException {
fs.delete(p, true);
assertFalse("output not cleaned up", fs.exists(p));
@@ -73,6 +60,7 @@ public class TestPipes extends TestCase
}
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
+ Path cppExamples = new Path(System.getProperty("install.c++.examples"));
Path inputPath = new Path("/testing/in");
Path outputPath = new Path("/testing/out");
try {
@@ -81,15 +69,17 @@ public class TestPipes extends TestCase
dfs = new MiniDFSCluster(conf, numSlaves, true, null);
mr = new MiniMRCluster(numSlaves, dfs.getFileSystem().getName(), 1);
writeInputFile(dfs.getFileSystem(), inputPath);
- runProgram(mr, dfs, wordCountSimple,
- inputPath, outputPath, 3, 2, twoSplitOutput, null);
+ runProgram(mr, dfs, new Path(cppExamples, "bin/wordcount-simple"),
+ inputPath, outputPath, 3, 2, twoSplitOutput);
cleanup(dfs.getFileSystem(), outputPath);
- runProgram(mr, dfs, wordCountSimple,
- inputPath, outputPath, 3, 0, noSortOutput, null);
+
+ runProgram(mr, dfs, new Path(cppExamples, "bin/wordcount-simple"),
+ inputPath, outputPath, 3, 0, noSortOutput);
cleanup(dfs.getFileSystem(), outputPath);
- runProgram(mr, dfs, wordCountPart,
- inputPath, outputPath, 3, 2, fixedPartitionOutput, null);
- runNonPipedProgram(mr, dfs, wordCountNoPipes, null);
+
+ runProgram(mr, dfs, new Path(cppExamples, "bin/wordcount-part"),
+ inputPath, outputPath, 3, 2, fixedPartitionOutput);
+ runNonPipedProgram(mr, dfs, new
Path(cppExamples,"bin/wordcount-nopipe"));
mr.waitUntilIdle();
} finally {
mr.shutdown();
@@ -97,7 +87,6 @@ public class TestPipes extends TestCase
}
}
-
final static String[] twoSplitOutput = new String[] {
"`and\t1\na\t1\nand\t1\nbeginning\t1\nbook\t1\nbut\t1\nby\t1\n" +
"conversation?'\t1\ndo:\t1\nhad\t2\nhaving\t1\nher\t2\nin\t1\nit\t1\n"+
@@ -135,7 +124,7 @@ public class TestPipes extends TestCase
""
};
- static void writeInputFile(FileSystem fs, Path dir) throws IOException {
+ private void writeInputFile(FileSystem fs, Path dir) throws IOException {
DataOutputStream out = fs.create(new Path(dir, "part0"));
out.writeBytes("Alice was beginning to get very tired of sitting by
her\n");
out.writeBytes("sister on the bank, and of having nothing to do: once\n");
@@ -146,18 +135,12 @@ public class TestPipes extends TestCase
out.close();
}
- static void runProgram(MiniMRCluster mr, MiniDFSCluster dfs,
+ private void runProgram(MiniMRCluster mr, MiniDFSCluster dfs,
Path program, Path inputPath, Path outputPath,
- int numMaps, int numReduces, String[]
expectedResults,
- JobConf conf
+ int numMaps, int numReduces, String[] expectedResults
) throws IOException {
Path wordExec = new Path("/testing/bin/application");
- JobConf job = null;
- if(conf == null) {
- job = mr.createJobConf();
- }else {
- job = new JobConf(conf);
- }
+ JobConf job = mr.createJobConf();
job.setNumMapTasks(numMaps);
job.setNumReduceTasks(numReduces);
{
@@ -216,21 +199,15 @@ public class TestPipes extends TestCase
* @param program the program to run
* @throws IOException
*/
- static void runNonPipedProgram(MiniMRCluster mr, MiniDFSCluster dfs,
- Path program, JobConf conf) throws
IOException {
- JobConf job;
- if(conf == null) {
- job = mr.createJobConf();
- }else {
- job = new JobConf(conf);
- }
-
+ private void runNonPipedProgram(MiniMRCluster mr, MiniDFSCluster dfs,
+ Path program) throws IOException {
+ JobConf job = mr.createJobConf();
job.setInputFormat(WordCountInputFormat.class);
FileSystem local = FileSystem.getLocal(job);
Path testDir = new Path("file:" + System.getProperty("test.build.data"),
"pipes");
Path inDir = new Path(testDir, "input");
- nonPipedOutDir = new Path(testDir, "output");
+ Path outDir = new Path(testDir, "output");
Path wordExec = new Path("/testing/bin/application");
Path jobXml = new Path(testDir, "job.xml");
{
@@ -251,21 +228,21 @@ public class TestPipes extends TestCase
out.writeBytes("hall silly cats drink java\n");
out.writeBytes("all dogs bow wow\n");
out.writeBytes("hello drink java\n");
- local.delete(nonPipedOutDir, true);
- local.mkdirs(nonPipedOutDir, new FsPermission(FsAction.ALL, FsAction.ALL,
- FsAction.ALL));
out.close();
+ local.delete(outDir, true);
+ local.mkdirs(outDir);
out = local.create(jobXml);
job.writeXml(out);
out.close();
- System.err.println("About to run: Submitter -conf " + jobXml + " -input "
- + inDir + " -output " + nonPipedOutDir + " -program "
- + dfs.getFileSystem().makeQualified(wordExec));
+ System.err.println("About to run: Submitter -conf " + jobXml +
+ " -input " + inDir + " -output " + outDir +
+ " -program " +
+ dfs.getFileSystem().makeQualified(wordExec));
try {
int ret = ToolRunner.run(new Submitter(),
new String[]{"-conf", jobXml.toString(),
"-input", inDir.toString(),
- "-output", nonPipedOutDir.toString(),
+ "-output", outDir.toString(),
"-program",
dfs.getFileSystem().makeQualified(wordExec).toString(),
"-reduces", "2"});
