Author: hairong
Date: Fri Dec 5 11:26:25 2008
New Revision: 723831
URL: http://svn.apache.org/viewvc?rev=723831&view=rev
Log:
HADOOP-4717. Removal of default port# in NameNode.getUri() causes a map/reduce
job failed to prompt temporary output. Contributed by Hairong Kuang.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=723831&r1=723830&r2=723831&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Dec 5 11:26:25 2008
@@ -1365,6 +1365,9 @@
HADOOP-4746. Job output directory should be normalized. (hairong)
+ HADOOP-4717. Removal of default port# in NameNode.getUri() causes a
+ map/reduce job failed to prompt temporary output. (hairong)
+
Release 0.18.2 - 2008-11-03
BUG FIXES
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=723831&r1=723830&r2=723831&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
Fri Dec 5 11:26:25 2008
@@ -92,6 +92,24 @@
super.checkPath(path);
}
+ /** Normalize paths that explicitly specify the default port. */
+ public Path makeQualified(Path path) {
+ URI thisUri = this.getUri();
+ URI thatUri = path.toUri();
+ String thatAuthority = thatUri.getAuthority();
+ if (thatUri.getScheme() != null
+ && thatUri.getScheme().equalsIgnoreCase(thisUri.getScheme())
+ && thatUri.getPort() == NameNode.DEFAULT_PORT
+ && thisUri.getPort() == -1
+ && thatAuthority.substring(0,thatAuthority.indexOf(":"))
+ .equalsIgnoreCase(thisUri.getAuthority())) {
+ path = new Path(thisUri.getScheme(), thisUri.getAuthority(),
+ thatUri.getPath());
+ }
+ return super.makeQualified(path);
+ }
+
+
public Path getWorkingDirectory() {
return workingDir;
}
Modified:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java?rev=723831&r1=723830&r2=723831&view=diff
==============================================================================
---
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
(original)
+++
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
Fri Dec 5 11:26:25 2008
@@ -141,8 +141,13 @@
}
private Path getFinalPath(Path jobOutputDir, Path taskOutput,
- Path taskOutputPath) {
- URI relativePath = taskOutputPath.toUri().relativize(taskOutput.toUri());
+ Path taskOutputPath) throws IOException {
+ URI taskOutputUri = taskOutput.toUri();
+ URI relativePath = taskOutputPath.toUri().relativize(taskOutputUri);
+ if (taskOutputUri == relativePath) {//taskOutputPath is not a parent of
taskOutput
+ throw new IOException("Can not get the relative path: base = " +
+ taskOutputPath + " child = " + taskOutput);
+ }
if (relativePath.getPath().length() > 0) {
return new Path(jobOutputDir, relativePath.getPath());
} else {
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java?rev=723831&r1=723830&r2=723831&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
Fri Dec 5 11:26:25 2008
@@ -33,6 +33,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.examples.WordCount;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -252,4 +253,37 @@
}
}
+ public void testWithDFSWithDefaultPort() throws IOException {
+ MiniDFSCluster dfs = null;
+ MiniMRCluster mr = null;
+ FileSystem fileSys = null;
+ try {
+ final int taskTrackers = 4;
+
+ Configuration conf = new Configuration();
+ // start a dfs with the default port number
+ dfs = new MiniDFSCluster(
+ NameNode.DEFAULT_PORT, conf, 4, true, true, null, null);
+ fileSys = dfs.getFileSystem();
+ mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1);
+
+ JobConf jobConf = mr.createJobConf();
+ TestResult result;
+ final Path inDir = new Path("./wc/input");
+ final Path outDir = new Path("hdfs://" +
+ dfs.getNameNode().getNameNodeAddress().getHostName() +
+ ":" + NameNode.DEFAULT_PORT +"/./wc/output");
+ String input = "The quick brown fox\nhas many silly\nred fox sox\n";
+ result = launchWordCount(jobConf, inDir, outDir, input, 3, 1);
+ assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
+ "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output);
+ } catch (java.net.BindException be) {
+ LOG.info("Skip the test this time because can not start namenode on port
"
+ + NameNode.DEFAULT_PORT, be);
+ } finally {
+ if (dfs != null) { dfs.shutdown(); }
+ if (mr != null) { mr.shutdown();
+ }
+ }
+ }
}
