Author: cutting
Date: Mon Jan 22 14:14:25 2007
New Revision: 498829
URL: http://svn.apache.org/viewvc?view=rev&rev=498829
Log:
HADOOP-901. Add support for recursive renaming to the S3 filesystem.
Contributed by Tom White.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/FileSystemStore.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=498829&r1=498828&r2=498829
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jan 22 14:14:25 2007
@@ -54,6 +54,9 @@
16. HADOOP-908. Add a new contrib package, Abacus, that simplifies
counting and aggregation, built on MapReduce. (Runping Qi via cutting)
+17. HADOOP-901. Add support for recursive renaming to the S3 filesystem.
+ (Tom White via cutting)
+
Release 0.10.1 - 2007-01-10
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/FileSystemStore.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/FileSystemStore.java?view=diff&rev=498829&r1=498828&r2=498829
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/FileSystemStore.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/FileSystemStore.java
Mon Jan 22 14:14:25 2007
@@ -25,6 +25,7 @@
void deleteBlock(Block block) throws IOException;
Set<Path> listSubPaths(Path path) throws IOException;
+ Set<Path> listDeepSubPaths(Path path) throws IOException;
/**
* Delete everything. Used for testing.
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java?view=diff&rev=498829&r1=498828&r2=498829
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
Mon Jan 22 14:14:25 2007
@@ -188,6 +188,27 @@
throw new S3Exception(e);
}
}
+
+ public Set<Path> listDeepSubPaths(Path path) throws IOException {
+ try {
+ String prefix = pathToKey(path);
+ if (!prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
+ }
+ S3Object[] objects = s3Service.listObjects(bucket, prefix, null);
+ Set<Path> prefixes = new TreeSet<Path>();
+ for (int i = 0; i < objects.length; i++) {
+ prefixes.add(keyToPath(objects[i].getKey()));
+ }
+ prefixes.remove(path);
+ return prefixes;
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
private void put(String key, InputStream in, long length) throws IOException
{
try {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?view=diff&rev=498829&r1=498828&r2=498829
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Mon
Jan 22 14:14:25 2007
@@ -174,15 +174,48 @@
@Override
public boolean renameRaw(Path src, Path dst) throws IOException {
- // TODO: Check corner cases: dst already exists,
- // or if path is directory with children
Path absoluteSrc = makeAbsolute(src);
- INode inode = store.getINode(absoluteSrc);
- if (inode == null) {
- throw new IOException("No such file.");
+ INode srcINode = store.getINode(absoluteSrc);
+ if (srcINode == null) {
+ // src path doesn't exist
+ return false;
+ }
+ Path absoluteDst = makeAbsolute(dst);
+ INode dstINode = store.getINode(absoluteDst);
+ if (dstINode != null && dstINode.isDirectory()) {
+ absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+ dstINode = store.getINode(absoluteDst);
+ }
+ if (dstINode != null) {
+ // dst path already exists - can't overwrite
+ return false;
+ }
+ Path dstParent = absoluteDst.getParent();
+ if (dstParent != null) {
+ INode dstParentINode = store.getINode(dstParent);
+ if (dstParentINode == null || dstParentINode.isFile()) {
+ // dst parent doesn't exist or is a file
+ return false;
+ }
+ }
+ return renameRawRecursive(absoluteSrc, absoluteDst);
+ }
+
+ private boolean renameRawRecursive(Path src, Path dst) throws IOException {
+ INode srcINode = store.getINode(src);
+ store.storeINode(dst, srcINode);
+ store.deleteINode(src);
+ if (srcINode.isDirectory()) {
+ for (Path oldSrc : store.listDeepSubPaths(src)) {
+ INode inode = store.getINode(oldSrc);
+ if (inode == null) {
+ return false;
+ }
+ Path newDst = new Path(oldSrc.toString().replaceFirst(src.toString(),
dst.toString()));
+ store.storeINode(newDst, inode);
+ store.deleteINode(oldSrc);
+ }
}
- store.storeINode(makeAbsolute(dst), inode);
- store.deleteINode(absoluteSrc);
return true;
}
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java?view=diff&rev=498829&r1=498828&r2=498829
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
Mon Jan 22 14:14:25 2007
@@ -66,6 +66,21 @@
return subPaths;
}
+ public Set<Path> listDeepSubPaths(Path path) throws IOException {
+ String pathString = path.toUri().getPath();
+ if (!pathString.endsWith("/")) {
+ pathString += "/";
+ }
+ // This is inefficient but more than adequate for testing purposes.
+ Set<Path> subPaths = new LinkedHashSet<Path>();
+ for (Path p : inodes.tailMap(path).keySet()) {
+ if (p.toUri().getPath().startsWith(pathString)) {
+ subPaths.add(p);
+ }
+ }
+ return subPaths;
+ }
+
public void storeINode(Path path, INode inode) throws IOException {
inodes.put(path, inode);
}
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java?view=diff&rev=498829&r1=498828&r2=498829
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Mon Jan 22 14:14:25 2007
@@ -228,32 +228,96 @@
assertFalse("file2 exists", s3FileSystem.exists(file2));
}
-
- public void testRename() throws Exception {
- int len = BLOCK_SIZE;
+
+ public void testRenameNonExistentPath() throws Exception {
+ Path src = new Path("/test/hadoop/path");
+ Path dst = new Path("/test/new/newpath");
+ rename(src, dst, false, false, false);
+ }
+
+ public void testRenameFileMoveToNonExistentDirectory() throws Exception {
+ Path src = new Path("/test/hadoop/file");
+ createEmptyFile(src);
+ Path dst = new Path("/test/new/newfile");
+ rename(src, dst, false, true, false);
+ }
+
+ public void testRenameFileMoveToExistingDirectory() throws Exception {
+ Path src = new Path("/test/hadoop/file");
+ createEmptyFile(src);
+ Path dst = new Path("/test/new/newfile");
+ s3FileSystem.mkdirs(dst.getParent());
+ rename(src, dst, true, false, true);
+ }
+
+ public void testRenameFileAsExistingFile() throws Exception {
+ Path src = new Path("/test/hadoop/file");
+ createEmptyFile(src);
+ Path dst = new Path("/test/new/newfile");
+ createEmptyFile(dst);
+ rename(src, dst, false, true, true);
+ }
+
+ public void testRenameFileAsExistingDirectory() throws Exception {
+ Path src = new Path("/test/hadoop/file");
+ createEmptyFile(src);
+ Path dst = new Path("/test/new/newdir");
+ s3FileSystem.mkdirs(dst);
+ rename(src, dst, true, false, true);
+ assertTrue("Destination changed", s3FileSystem.exists(new
Path("/test/new/newdir/file")));
+ }
+
+ public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception
{
+ Path src = new Path("/test/hadoop/dir");
+ s3FileSystem.mkdirs(src);
+ Path dst = new Path("/test/new/newdir");
+ rename(src, dst, false, true, false);
+ }
+
+ public void testRenameDirectoryMoveToExistingDirectory() throws Exception {
+ Path src = new Path("/test/hadoop/dir");
+ s3FileSystem.mkdirs(src);
+ createEmptyFile(new Path("/test/hadoop/dir/file1"));
+ createEmptyFile(new Path("/test/hadoop/dir/subdir/file2"));
- Path path = new Path("/test/hadoop/file");
+ Path dst = new Path("/test/new/newdir");
+ s3FileSystem.mkdirs(dst.getParent());
+ rename(src, dst, true, false, true);
- s3FileSystem.mkdirs(path.getParent());
-
- createEmptyFile(path);
-
- assertTrue("Exists", s3FileSystem.exists(path));
-
- Path newPath = new Path("/test/hadoop/newfile");
- s3FileSystem.rename(path, newPath);
- assertFalse("No longer exists", s3FileSystem.exists(path));
- assertTrue("Moved", s3FileSystem.exists(newPath));
-
- FSInputStream in = s3FileSystem.openRaw(newPath);
- byte[] buf = new byte[len];
+ assertFalse("Nested file1 exists", s3FileSystem.exists(new
Path("/test/hadoop/dir/file1")));
+ assertFalse("Nested file2 exists", s3FileSystem.exists(new
Path("/test/hadoop/dir/subdir/file2")));
+ assertTrue("Renamed nested file1 exists", s3FileSystem.exists(new
Path("/test/new/newdir/file1")));
+ assertTrue("Renamed nested exists", s3FileSystem.exists(new
Path("/test/new/newdir/subdir/file2")));
+ }
+
+ public void testRenameDirectoryAsExistingFile() throws Exception {
+ Path src = new Path("/test/hadoop/dir");
+ s3FileSystem.mkdirs(src);
+ Path dst = new Path("/test/new/newfile");
+ createEmptyFile(dst);
+ rename(src, dst, false, true, true);
+ }
+
+ public void testRenameDirectoryAsExistingDirectory() throws Exception {
+ Path src = new Path("/test/hadoop/dir");
+ s3FileSystem.mkdirs(src);
+ createEmptyFile(new Path("/test/hadoop/dir/file1"));
+ createEmptyFile(new Path("/test/hadoop/dir/subdir/file2"));
- in.readFully(0, buf);
-
- assertEquals(len, buf.length);
- for (int i = 0; i < buf.length; i++) {
- assertEquals("Position " + i, data[i], buf[i]);
- }
+ Path dst = new Path("/test/new/newdir");
+ s3FileSystem.mkdirs(dst);
+ rename(src, dst, true, false, true);
+ assertTrue("Destination changed", s3FileSystem.exists(new
Path("/test/new/newdir/dir")));
+ assertFalse("Nested file1 exists", s3FileSystem.exists(new
Path("/test/hadoop/dir/file1")));
+ assertFalse("Nested file2 exists", s3FileSystem.exists(new
Path("/test/hadoop/dir/subdir/file2")));
+ assertTrue("Renamed nested file1 exists", s3FileSystem.exists(new
Path("/test/new/newdir/dir/file1")));
+ assertTrue("Renamed nested exists", s3FileSystem.exists(new
Path("/test/new/newdir/dir/subdir/file2")));
+ }
+
+ private void rename(Path src, Path dst, boolean renameSucceeded, boolean
srcExists, boolean dstExists) throws IOException {
+ assertEquals("Rename result", renameSucceeded, s3FileSystem.rename(src,
dst));
+ assertEquals("Source exists", srcExists, s3FileSystem.exists(src));
+ assertEquals("Destination exists", dstExists, s3FileSystem.exists(dst));
}
private void createEmptyFile(Path path) throws IOException {
