Author: tomwhite
Date: Sun Sep 23 14:30:24 2007
New Revision: 578605
URL: http://svn.apache.org/viewvc?rev=578605&view=rev
Log:
HADOOP-1783. Make S3 FileSystem return Paths fully-qualified with scheme and
host.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=578605&r1=578604&r2=578605&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Sun Sep 23 14:30:24 2007
@@ -176,6 +176,9 @@
HADOOP-1882. Remove spurious asterisks from decimal number displays.
(Raghu Angadi via cutting)
+ HADOOP-1783. Make S3 FileSystem return Paths fully-qualified with
+ scheme and host. (tomwhite)
+
IMPROVEMENTS
HADOOP-1266. Remove dependency of package org.apache.hadoop.net on
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=578605&r1=578604&r2=578605&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Sun
Sep 23 14:30:24 2007
@@ -173,7 +173,7 @@
}
ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
for (Path p : store.listSubPaths(absolutePath)) {
- ret.add(getFileStatus(p));
+ ret.add(getFileStatus(p.makeQualified(this)));
}
return ret.toArray(new FileStatus[0]);
}
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java?rev=578605&r1=578604&r2=578605&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
Sun Sep 23 14:30:24 2007
@@ -56,7 +56,7 @@
}
public void deleteINode(Path path) throws IOException {
- inodes.remove(path);
+ inodes.remove(normalize(path));
}
public void deleteBlock(Block block) throws IOException {
@@ -64,7 +64,7 @@
}
public boolean inodeExists(Path path) throws IOException {
- return inodes.containsKey(path);
+ return inodes.containsKey(normalize(path));
}
public boolean blockExists(long blockId) throws IOException {
@@ -72,7 +72,7 @@
}
public INode retrieveINode(Path path) throws IOException {
- return inodes.get(path);
+ return inodes.get(normalize(path));
}
public File retrieveBlock(Block block, long byteRangeStart) throws
IOException {
@@ -101,10 +101,11 @@
}
public Set<Path> listSubPaths(Path path) throws IOException {
+ Path normalizedPath = normalize(path);
// This is inefficient but more than adequate for testing purposes.
Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(path).keySet()) {
- if (path.equals(p.getParent())) {
+ for (Path p : inodes.tailMap(normalizedPath).keySet()) {
+ if (normalizedPath.equals(p.getParent())) {
subPaths.add(p);
}
}
@@ -112,13 +113,14 @@
}
public Set<Path> listDeepSubPaths(Path path) throws IOException {
- String pathString = path.toUri().getPath();
+ Path normalizedPath = normalize(path);
+ String pathString = normalizedPath.toUri().getPath();
if (!pathString.endsWith("/")) {
pathString += "/";
}
// This is inefficient but more than adequate for testing purposes.
Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(path).keySet()) {
+ for (Path p : inodes.tailMap(normalizedPath).keySet()) {
if (p.toUri().getPath().startsWith(pathString)) {
subPaths.add(p);
}
@@ -127,7 +129,7 @@
}
public void storeINode(Path path, INode inode) throws IOException {
- inodes.put(path, inode);
+ inodes.put(normalize(path), inode);
}
public void storeBlock(Block block, File file) throws IOException {
@@ -146,6 +148,13 @@
}
}
blocks.put(block.getId(), out.toByteArray());
+ }
+
+ private Path normalize(Path path) {
+ if (!path.isAbsolute()) {
+ throw new IllegalArgumentException("Path must be absolute: " + path);
+ }
+ return new Path(path.toUri().getPath());
}
public void purge() throws IOException {
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java?rev=578605&r1=578604&r2=578605&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Sun Sep 23 14:30:24 2007
@@ -111,17 +111,21 @@
Path[] paths = s3FileSystem.listPaths(new Path("/"));
assertEquals(1, paths.length);
- assertEquals(new Path("/test"), paths[0]);
+ assertEquals(new Path("/test").makeQualified(s3FileSystem), paths[0]);
paths = s3FileSystem.listPaths(new Path("/test"));
assertEquals(1, paths.length);
- assertEquals(new Path("/test/hadoop"), paths[0]);
+ assertEquals(new Path("/test/hadoop").makeQualified(s3FileSystem),
+ paths[0]);
paths = s3FileSystem.listPaths(new Path("/test/hadoop"));
assertEquals(3, paths.length);
- assertEquals(new Path("/test/hadoop/a"), paths[0]);
- assertEquals(new Path("/test/hadoop/b"), paths[1]);
- assertEquals(new Path("/test/hadoop/c"), paths[2]);
+ assertEquals(new Path("/test/hadoop/a").makeQualified(s3FileSystem),
+ paths[0]);
+ assertEquals(new Path("/test/hadoop/b").makeQualified(s3FileSystem),
+ paths[1]);
+ assertEquals(new Path("/test/hadoop/c").makeQualified(s3FileSystem),
+ paths[2]);
paths = s3FileSystem.listPaths(new Path("/test/hadoop/a"));
assertEquals(0, paths.length);
