Author: cutting
Date: Fri Feb 16 13:44:19 2007
New Revision: 508597
URL: http://svn.apache.org/viewvc?view=rev&rev=508597
Log:
HADOOP-1021. Fix MRCaching-based unit tests on Windows. Contributed by Nigel.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Feb 16 13:44:19 2007
@@ -59,6 +59,9 @@
17. HADOOP-803. Reduce memory use by HDFS namenode, phase I.
(Raghu Angadi via cutting)
+18. HADOOP-1021. Fix MRCaching-based unit tests on Windows.
+ (Nigel Daley via cutting)
+
Branch 0.11 (unreleased)
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java Fri
Feb 16 13:44:19 2007
@@ -60,18 +60,17 @@
try {
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
- FileSystem fs = FileSystem.get(conf);
// read the cached files (unzipped, unjarred and text)
// and put it into a single file TEST_ROOT_DIR/test.txt
String TEST_ROOT_DIR = jconf.get("test.build.data","/tmp");
- Path file = new Path(TEST_ROOT_DIR);
+ Path file = new Path("file:///",TEST_ROOT_DIR);
+ FileSystem fs = FileSystem.getLocal(conf);
if (!fs.mkdirs(file)) {
throw new IOException("Mkdirs failed to create " + file.toString());
}
Path fileOut = new Path(file, "test.txt");
fs.delete(fileOut);
DataOutputStream out = fs.create(fileOut);
-
for (int i = 0; i < localArchives.length; i++) {
// read out the files from these archives
File f = new File(localArchives[i].toString());
@@ -126,10 +125,11 @@
}
public static boolean launchMRCache(String indir,
- String outdir, JobConf conf, String input)
+ String outdir, String cacheDir, JobConf conf, String input)
throws IOException {
String TEST_ROOT_DIR = new
Path(System.getProperty("test.build.data","/tmp"))
.toString().replace(' ', '+');
+ //if (TEST_ROOT_DIR.startsWith("C:")) TEST_ROOT_DIR = "/tmp";
conf.set("test.build.data",TEST_ROOT_DIR);
final Path inDir = new Path(indir);
final Path outDir = new Path(outdir);
@@ -139,6 +139,7 @@
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
{
+System.out.println("HERE:"+inDir);
DataOutputStream file = fs.create(new Path(inDir, "part-0"));
file.writeBytes(input);
file.close();
@@ -162,19 +163,28 @@
Path txtPath = new Path(localPath, new Path("test.txt"));
Path jarPath = new Path(localPath, new Path("test.jar"));
Path zipPath = new Path(localPath, new Path("test.zip"));
- Path cacheTest = new Path(TEST_ROOT_DIR + "/cachedir");
- fs.delete(cacheTest);
- if (!fs.mkdirs(cacheTest)) {
- throw new IOException("Mkdirs failed to create " + cacheTest.toString());
- }
- fs.copyFromLocalFile(txtPath, cacheTest);
- fs.copyFromLocalFile(jarPath, cacheTest);
- fs.copyFromLocalFile(zipPath, cacheTest);
+ Path cachePath = new Path(cacheDir);
+ fs.delete(cachePath);
+ if (!fs.mkdirs(cachePath)) {
+ throw new IOException("Mkdirs failed to create " + cachePath.toString());
+ }
+ fs.copyFromLocalFile(txtPath, cachePath);
+ fs.copyFromLocalFile(jarPath, cachePath);
+ fs.copyFromLocalFile(zipPath, cachePath);
// setting the cached archives to zip, jar and simple text files
String fileSys = fs.getName();
- String archive1 = "dfs://" + fileSys + TEST_ROOT_DIR +
"/cachedir/test.jar";
- String archive2 = "dfs://" + fileSys + TEST_ROOT_DIR +
"/cachedir/test.zip";
- String file1 = "dfs://" + fileSys + TEST_ROOT_DIR + "/cachedir/test.txt";
+ String archive1;
+ String archive2;
+ String file1;
+ if (fileSys.equals("local")) {
+ archive1 = "file://" + cachePath + "/test.jar";
+ archive2 = "file://" + cachePath + "/test.zip";
+ file1 = "file://" + cachePath + "/test.txt";
+ } else {
+ archive1 = "dfs://" + fileSys + cachePath + "/test.jar";
+ archive2 = "dfs://" + fileSys + cachePath + "/test.zip";
+ file1 = "dfs://" + fileSys + cachePath + "/test.txt";
+ }
URI uri1 = null;
URI uri2 = null;
URI uri3 = null;
@@ -193,8 +203,8 @@
// match the real string. check if there are 3 instances or not.
Path result = new Path(TEST_ROOT_DIR + "/test.txt");
{
- BufferedReader file = new BufferedReader(new InputStreamReader(fs
- .open(result)));
+ BufferedReader file = new BufferedReader(new InputStreamReader(
+ FileSystem.getLocal(conf).open(result)));
String line = file.readLine();
while (line != null) {
if (!testStr.equals(line))
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
Fri Feb 16 13:44:19 2007
@@ -42,6 +42,7 @@
// run the wordcount example with caching
boolean ret = MRCaching.launchMRCache("/testing/wc/input",
"/testing/wc/output",
+ "/cachedir",
mr.createJobConf(),
"The quick brown fox\nhas many
silly\n"
+ "red fox sox\n");
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
Fri Feb 16 13:44:19 2007
@@ -47,6 +47,7 @@
JobConf job = mr.createJobConf();
boolean ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input",
TEST_ROOT_DIR + "/wc/output",
+ TEST_ROOT_DIR + "/cachedir",
job,
"The quick brown fox\nhas many
silly\n"
+ "red fox sox\n");
