Author: cutting
Date: Tue Jul 17 13:15:02 2007
New Revision: 557044
URL: http://svn.apache.org/viewvc?view=rev&rev=557044
Log:
HADOOP-1620. Reduce the number of abstract FileSystem methods, simplifying
implementations.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Jul 17 13:15:02 2007
@@ -373,6 +373,9 @@
116. HADOOP-1564. Add unit tests for HDFS block-level checksums.
(Dhruba Borthakur via cutting)
+117. HADOOP-1620. Reduce the number of abstract FileSystem methods,
+ simplifying implementations. (cutting)
+
Release 0.13.0 - 2007-06-08
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
Tue Jul 17 13:15:02 2007
@@ -198,31 +198,6 @@
dfs.release(getPath(f));
}
- @Override
- public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {
- FileUtil.copy(localFs, src, this, dst, delSrc, getConf());
- }
-
- @Override
- public void copyToLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {
- FileUtil.copy(this, src, localFs, dst, delSrc, getConf());
- }
-
- public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- return tmpLocalFile;
- }
-
- /**
- * Move completed local data to DFS destination
- */
- public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- moveFromLocalFile(tmpLocalFile, fsOutputFile);
- }
-
public void close() throws IOException {
super.close();
dfs.close();
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java
Tue Jul 17 13:15:02 2007
@@ -168,7 +168,7 @@
Configuration conf, boolean isArchive,
String md5, Path currentWorkDir) throws IOException {
boolean b = true;
boolean doSymlink = getSymlink(conf);
- FileSystem dfs = getFileSystem(cache, conf);
+ FileSystem dfs = FileSystem.get(cache, conf);
b = ifExistsAndFresh(cacheStatus, cache, dfs, md5, conf);
String link = currentWorkDir.toString() + Path.SEPARATOR +
cache.getFragment();
File flink = new File(link);
@@ -290,7 +290,7 @@
byte[] b = new byte[CRC_BUFFER_SIZE];
byte[] digest = null;
- FileSystem fileSystem = getFileSystem(cache, conf);
+ FileSystem fileSystem = FileSystem.get(cache, conf);
if (!(fileSystem instanceof ChecksumFileSystem)) {
throw new IOException("Not a checksummed file system: "
+fileSystem.getUri());
@@ -357,26 +357,6 @@
new File(workDir, list[i].getName()).toString());
}
}
- }
-
- private static String getFileSysName(URI url) {
- String fsname = url.getScheme();
- if ("hdfs".equals(fsname)) {
- String host = url.getHost();
- int port = url.getPort();
- return (port == (-1)) ? host : (host + ":" + port);
- } else {
- return null;
- }
- }
-
- private static FileSystem getFileSystem(URI cache, Configuration conf)
- throws IOException {
- String fileSysName = getFileSysName(cache);
- if (fileSysName != null)
- return FileSystem.getNamed(fileSysName, conf);
- else
- return FileSystem.get(conf);
}
/**
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Tue Jul
17 13:15:02 2007
@@ -87,7 +87,7 @@
/** Returns the configured filesystem implementation.*/
public static FileSystem get(Configuration conf) throws IOException {
- return getNamed(conf.get("fs.default.name", "local"), conf);
+ return getNamed(conf.get("fs.default.name", "file:///"), conf);
}
/** Called after a new FileSystem instance is constructed.
@@ -102,7 +102,7 @@
public abstract URI getUri();
/** @deprecated call #getUri() instead.*/
- public abstract String getName();
+ public String getName() { return getUri().toString(); }
/** @deprecated call #get(URI,Configuration) instead. */
public static FileSystem getNamed(String name, Configuration conf)
@@ -216,7 +216,17 @@
*
* The FileSystem will simply return an elt containing 'localhost'.
*/
- public abstract String[][] getFileCacheHints(Path f, long start, long len)
throws IOException;
+ public String[][] getFileCacheHints(Path f, long start, long len)
+ throws IOException {
+ if (!exists(f)) {
+ return null;
+ } else {
+ String result[][] = new String[1][];
+ result[0] = new String[1];
+ result[0][0] = "localhost";
+ return result;
+ }
+ }
/**
* Opens an FSDataInputStream at the indicated Path.
@@ -387,7 +397,10 @@
* @return true if successful;
* false if file does not exist or is a directory
*/
- public abstract boolean setReplication(Path src, short replication) throws
IOException;
+ public boolean setReplication(Path src, short replication)
+ throws IOException {
+ return true;
+ }
/**
* Renames Path src to Path dst. Can take place on local fs
@@ -725,7 +738,7 @@
* @deprecated FS does not support file locks anymore.
*/
@Deprecated
- public abstract void lock(Path f, boolean shared) throws IOException;
+ public void lock(Path f, boolean shared) throws IOException {}
/**
* Release the lock
@@ -733,7 +746,7 @@
* @deprecated FS does not support file locks anymore.
*/
@Deprecated
- public abstract void release(Path f) throws IOException;
+ public void release(Path f) throws IOException {}
/**
* The src file is on the local disk. Add it to FS at
@@ -758,8 +771,10 @@
* the given dst name.
* delSrc indicates if the source should be removed
*/
- public abstract void copyFromLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException;
+ public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
+ throws IOException {
+ FileUtil.copy(getLocal(getConf()), src, this, dst, delSrc, getConf());
+ }
/**
* The src file is under FS, and the dst is on the local disk.
@@ -783,8 +798,10 @@
* Copy it from FS control to the local dst name.
* delSrc indicates if the src will be removed or not.
*/
- public abstract void copyToLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException;
+ public void copyToLocalFile(boolean delSrc, Path src, Path dst)
+ throws IOException {
+ FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf());
+ }
/**
* Returns a local File that the user can write output to. The caller
@@ -792,7 +809,10 @@
* file. If the FS is local, we write directly into the target. If
* the FS is remote, we write into the tmp local area.
*/
- public abstract Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException;
+ public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
+ throws IOException {
+ return tmpLocalFile;
+ }
/**
* Called when we're all done writing to the target. A local FS will
@@ -800,7 +820,10 @@
* FS will copy the contents of tmpLocalFile to the correct target at
* fsOutputFile.
*/
- public abstract void completeLocalOutput(Path fsOutputFile, Path
tmpLocalFile) throws IOException;
+ public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
+ throws IOException {
+ moveFromLocalFile(tmpLocalFile, fsOutputFile);
+ }
/**
* No more filesystem operations are needed. Will
@@ -846,7 +869,7 @@
/**
* Get the default replication.
*/
- public abstract short getDefaultReplication();
+ public short getDefaultReplication() { return 1; }
/*
* Return a file status object that represents the
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
Tue Jul 17 13:15:02 2007
@@ -80,11 +80,6 @@
return uri;
}
- /** @deprecated */
- public String getName() {
- return uri.toString();
- }
-
/**
* Return 1x1 'inmemory' cell if the file exists.
* Return null if otherwise.
@@ -229,13 +224,6 @@
}
}
- /**
- * Replication is not supported for the inmemory file system.
- */
- public short getDefaultReplication() {
- return 1;
- }
-
public boolean setReplication(Path src, short replication)
throws IOException {
return true;
@@ -290,32 +278,6 @@
return true;
}
- /** lock operations are not supported */
- public void lock(Path f, boolean shared) throws IOException {}
- public void release(Path f) throws IOException {}
-
- /** copy/move operations are not supported */
- public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {
- }
-
- public void copyToLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {
- }
-
- public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- return fsOutputFile;
- }
-
- public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- }
-
- public long getDefaultBlockSize() {
- return 32 * 1024; //some random large number. can be anything actually
- }
-
public FileStatus getFileStatus(Path f) throws IOException {
synchronized (this) {
return new InMemoryFileStatus(pathToFileAttribs.get(getPath(f)));
@@ -441,21 +403,6 @@
public InMemoryFileSystem(URI uri, Configuration conf) {
super(new RawInMemoryFileSystem(uri, conf));
- }
-
- /** copy/move operations are not supported */
- public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {}
- public void copyToLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {}
-
- public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- return fsOutputFile;
- }
-
- public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
}
/**
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Tue Jul 17 13:15:02 2007
@@ -54,24 +54,9 @@
return new File(path.toUri().getPath());
}
- /**
- * Return 1x1 'localhost' cell if the file exists.
- * Return null if otherwise.
- */
- public String[][] getFileCacheHints(Path f, long start, long len) throws
IOException {
- if (!exists(f)) {
- return null;
- } else {
- String result[][] = new String[1][];
- result[0] = new String[1];
- result[0][0] = "localhost";
- return result;
- }
- }
-
/** @deprecated */
public String getName() { return "local"; }
-
+
public URI getUri() { return NAME; }
public void initialize(URI uri, Configuration conf) {
@@ -190,16 +175,6 @@
new BufferedOutputStream(new LocalFSFileOutputStream(f), bufferSize));
}
- /**
- * Replication is not supported for the local file system.
- */
- /** Set the replication of the given file */
- public boolean setReplication(Path src,
- short replication
- ) throws IOException {
- return true;
- }
-
public boolean rename(Path src, Path dst) throws IOException {
if (useCopyForRename) {
return FileUtil.copy(this, src, this, dst, true, getConf());
@@ -321,18 +296,6 @@
rename(src, dst);
}
- @Override
- public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {
- FileUtil.copy(this, src, this, dst, delSrc, getConf());
- }
-
- @Override
- public void copyToLocalFile(boolean delSrc, Path src, Path dst)
- throws IOException {
- FileUtil.copy(this, src, this, dst, delSrc, getConf());
- }
-
// We can write output directly to the final location
public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException {
@@ -356,10 +319,6 @@
return new RawLocalFileStatus(pathToFile(f));
}
- public short getDefaultReplication() {
- return 1;
- }
-
private class RawLocalFileStatus implements FileStatus {
private long length;
private boolean isDir;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Tue
Jul 17 13:15:02 2007
@@ -280,15 +280,6 @@
}
/**
- * Replication is not supported for S3 file systems since S3 handles it for
- * us.
- */
- @Override
- public short getDefaultReplication() {
- return 1;
- }
-
- /**
* FileStatus for S3 file systems.
*/
@Override
@@ -298,68 +289,6 @@
throw new IOException(f.toString() + ": No such file or directory.");
}
return new S3FileStatus(inode);
- }
-
- /**
- * Replication is not supported for S3 file systems since S3 handles it for
- * us.
- */
- @Override
- public boolean setReplication(Path path, short replication)
- throws IOException {
- return true;
- }
-
- @Override
- public long getDefaultBlockSize() {
- return getConf().getLong("fs.s3.block.size", DEFAULT_BLOCK_SIZE);
- }
-
- /**
- * Return 1x1 'localhost' cell if the file exists. Return null if otherwise.
- */
- @Override
- public String[][] getFileCacheHints(Path f, long start, long len)
- throws IOException {
- // TODO: Check this is the correct behavior
- if (!exists(f)) {
- return null;
- }
- return new String[][] { { "localhost" } };
- }
-
- /** @deprecated */ @Deprecated
- @Override
- public void lock(Path path, boolean shared) throws IOException {
- // TODO: Design and implement
- }
-
- /** @deprecated */ @Deprecated
- @Override
- public void release(Path path) throws IOException {
- // TODO: Design and implement
- }
-
- @Override
- public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws
IOException {
- FileUtil.copy(localFs, src, this, dst, delSrc, getConf());
- }
-
- @Override
- public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws
IOException {
- FileUtil.copy(this, src, localFs, dst, delSrc, getConf());
- }
-
- @Override
- public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- return tmpLocalFile;
- }
-
- @Override
- public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- moveFromLocalFile(tmpLocalFile, fsOutputFile);
}
// diagnostic methods
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java?view=diff&rev=557044&r1=557043&r2=557044
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
Tue Jul 17 13:15:02 2007
@@ -46,7 +46,7 @@
public void testWithLocal() throws IOException {
MiniMRCluster mr = null;
try {
- mr = new MiniMRCluster(2, "local", 3);
+ mr = new MiniMRCluster(2, "file:///", 3);
double estimate = PiEstimator.launch(NUM_MAPS, NUM_SAMPLES,
mr.createJobConf());
double error = Math.abs(Math.PI - estimate);
