Author: cutting
Date: Wed May 23 13:53:01 2007
New Revision: 541070
URL: http://svn.apache.org/viewvc?view=rev&rev=541070
Log:
HADOOP-1226. Change HDFS so that paths it returns are always fully qualified.
Contributed by Dhruba.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed May 23 13:53:01 2007
@@ -60,6 +60,9 @@
19. HADOOP-1413. Add example program that uses Knuth's dancing links
algorithm to solve pentomino problems. (omalley via cutting)
+ 20. HADOOP-1226. Change HDFS so that paths it returns are always
+ fully qualified. (Dhruba Borthakur via cutting)
+
Branch 0.13 (unreleased changes)
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java Wed May 23
13:53:01 2007
@@ -18,7 +18,7 @@
package org.apache.hadoop.dfs;
import org.apache.hadoop.fs.Path;
-
+import org.apache.hadoop.fs.FileSystem;
/**
* DfsPath is a Path that's been annotated with some extra information.
@@ -28,8 +28,11 @@
class DfsPath extends Path {
DFSFileInfo info;
- public DfsPath(DFSFileInfo info) {
- super(info.getPath());
+ /**
+ * DfsPaths are fully qualified with scheme and authority.
+ */
+ public DfsPath(DFSFileInfo info, FileSystem fs) {
+ super((new Path(info.getPath())).makeQualified(fs).toString());
this.info = info;
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
Wed May 23 13:53:01 2007
@@ -218,7 +218,7 @@
} else {
Path results[] = new DfsPath[info.length];
for (int i = 0; i < info.length; i++) {
- results[i] = new DfsPath(info[i]);
+ results[i] = new DfsPath(info[i], this);
}
return results;
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed May
23 13:53:01 2007
@@ -181,32 +181,7 @@
/** Make sure that a path specifies a FileSystem. */
public Path makeQualified(Path path) {
checkPath(path);
-
- if (!path.isAbsolute())
- path = new Path(getWorkingDirectory(), path);
-
- URI pathUri = path.toUri();
- URI fsUri = getUri();
-
- String scheme = pathUri.getScheme();
- String authority = pathUri.getAuthority();
-
- if (scheme != null &&
- (authority != null || fsUri.getAuthority() == null))
- return path;
-
- if (scheme == null) {
- scheme = fsUri.getScheme();
- }
-
- if (authority == null) {
- authority = fsUri.getAuthority();
- if (authority == null) {
- authority = "";
- }
- }
-
- return new Path(scheme+":"+"//"+authority + pathUri.getPath());
+ return path.makeQualified(this);
}
///////////////////////////////////////////////////////////////
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java Wed May 23
13:53:01 2007
@@ -329,7 +329,7 @@
}
for (int i = 0; i < items.length; i++) {
Path cur = items[i];
- System.out.println(cur + "\t"
+ System.out.println(cur.toUri().getPath() + "\t"
+ (fs.isDirectory(cur) ?
"<dir>" :
("<r " + fs.getReplication(cur)
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java Wed May 23
13:53:01 2007
@@ -264,5 +264,35 @@
}
return depth;
}
-
+
+ /** Returns a qualified path object. */
+ public Path makeQualified(FileSystem fs) {
+ Path path = this;
+ if (!isAbsolute()) {
+ path = new Path(fs.getWorkingDirectory(), this);
+ }
+
+ URI pathUri = path.toUri();
+ URI fsUri = fs.getUri();
+
+ String scheme = pathUri.getScheme();
+ String authority = pathUri.getAuthority();
+
+ if (scheme != null &&
+ (authority != null || fsUri.getAuthority() == null))
+ return path;
+
+ if (scheme == null) {
+ scheme = fsUri.getScheme();
+ }
+
+ if (authority == null) {
+ authority = fsUri.getAuthority();
+ if (authority == null) {
+ authority = "";
+ }
+ }
+
+ return new Path(scheme+":"+"//"+authority + pathUri.getPath());
+ }
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java Wed May 23
13:53:01 2007
@@ -159,7 +159,13 @@
public String toString() { return taskId; }
private Path getTaskOutputPath(JobConf conf) {
- return new Path(conf.getOutputPath(), ("_" + taskId));
+ Path p = new Path(conf.getOutputPath(), ("_" + taskId));
+ try {
+ return p.makeQualified(FileSystem.get(conf));
+ } catch (IOException ie) {
+ LOG.warn(StringUtils.stringifyException(ie));
+ return p;
+ }
}
/**
