Author: cutting
Date: Sat Apr 1 11:19:38 2006
New Revision: 390734
URL: http://svn.apache.org/viewcvs?rev=390734&view=rev
Log:
More fixes to get working directory to work on Windows. Long-term we should
stop using java.io.File for our abstract file paths. On Windows, 'new
File(/foo/bar).isAbsolute()' returns false, which caused lots of problems. So
I have added a FileSystem.isAbsolute() method that we use internally. I also
added a few more .getAbsoluteFile() calls to convert paths into absolute paths
so that LocalFileSystem works correctly. Finally I took advantage of file
status information cached in DFSFile, eliminating some namenode RPCs.
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Sat
Apr 1 11:19:38 2006
@@ -273,10 +273,10 @@
int hashCode = path.hashCode();
for (int i = 0; i < dirs.length; i++) { // try each local dir
int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
- File file = new File(dirs[index], path);
+ File file = new File(dirs[index], path).getAbsoluteFile();
File dir = file.getParentFile();
if (dir.exists() || dir.mkdirs()) {
- return file.getAbsoluteFile();
+ return file;
}
}
throw new IOException("No valid local directories in property: "+dirsProp);
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java Sat Apr 1
11:19:38 2006
@@ -57,6 +57,9 @@
public boolean isHidden() {
return false;
}
+ public boolean isAbsolute() {
+ return true;
+ }
/**
* We need to reimplement some of them
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
Sat Apr 1 11:19:38 2006
@@ -54,7 +54,7 @@
}
private File makeAbsolute(File f) {
- if (f.isAbsolute()) {
+ if (isAbsolute(f)) {
return f;
} else {
return new File(workingDir, f.getPath());
@@ -102,10 +102,23 @@
}
public boolean isDirectory(File f) throws IOException {
+ if (f instanceof DFSFile) {
+ return ((DFSFile)f).isDirectory();
+ }
return dfs.isDirectory(getPath(f));
}
+ public boolean isAbsolute(File f) {
+ return f.isAbsolute() ||
+ f.getPath().startsWith("/") ||
+ f.getPath().startsWith("\\");
+ }
+
public long getLength(File f) throws IOException {
+ if (f instanceof DFSFile) {
+ return ((DFSFile)f).length();
+ }
+
DFSFileInfo info[] = dfs.listFiles(getPath(f));
return info[0].getLen();
}
@@ -282,7 +295,7 @@
path.append(DFSFile.DFS_FILE_SEPARATOR);
path.append(l.get(i));
}
- if (f.isAbsolute() && path.length() == 0) {
+ if (isAbsolute(f) && path.length() == 0) {
path.append(DFSFile.DFS_FILE_SEPARATOR);
}
return path.toString();
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Sat Apr
1 11:19:38 2006
@@ -265,6 +265,9 @@
}
}
+ /** True iff the named path is absolute. */
+ public abstract boolean isAbsolute(File f);
+
/** The number of bytes in a file. */
public abstract long getLength(File f) throws IOException;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java Sat
Apr 1 11:19:38 2006
@@ -153,7 +153,7 @@
}
private File makeAbsolute(File f) {
- if (f.isAbsolute()) {
+ if (isAbsolute(f)) {
return f;
} else {
return new File(workingDir, f.toString());
@@ -197,6 +197,10 @@
public boolean isDirectory(File f) throws IOException {
f = makeAbsolute(f);
return f.isDirectory();
+ }
+
+ public boolean isAbsolute(File f) {
+ return f.isAbsolute();
}
public long getLength(File f) throws IOException {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java
Sat Apr 1 11:19:38 2006
@@ -65,7 +65,7 @@
for (int i = 0; i < dirs.length; i++) {
// if it is relative, make it absolute using the directory from the
// JobConf
- if (workDir != null && !dirs[i].isAbsolute()) {
+ if (workDir != null && !fs.isAbsolute(dirs[i])) {
dirs[i] = new File(workDir, dirs[i].toString());
}
File[] dir = fs.listFiles(dirs[i]);
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Sat Apr
1 11:19:38 2006
@@ -99,8 +99,8 @@
public void setJar(String jar) { set("mapred.jar", jar); }
public File getSystemDir() {
- return new File(get("mapred.system.dir",
- "/tmp/hadoop/mapred/system"));
+ return new File(get("mapred.system.dir", "/tmp/hadoop/mapred/system"))
+ .getAbsoluteFile();
}
public String[] getLocalDirs() throws IOException {
