Author: tomwhite
Date: Mon Sep 8 03:47:40 2008
New Revision: 693048
URL: http://svn.apache.org/viewvc?rev=693048&view=rev
Log:
HADOOP-3361. Implement renames for NativeS3FileSystem. Contributed by Albert
Chern.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 8 03:47:40 2008
@@ -279,6 +279,9 @@
HADOOP-3498. File globbing alternation should be able to span path
components. (tomwhite)
+ HADOOP-3361. Implement renames for NativeS3FileSystem.
+ (Albert Chern via tomwhite)
+
OPTIMIZATIONS
HADOOP-3556. Removed lock contention in MD5Hash by changing the
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
---
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
(original)
+++
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
Mon Sep 8 03:47:40 2008
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.s3native;
+import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
+
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -171,17 +173,27 @@
throws IOException {
return list(prefix, maxListingLength, null);
}
-
+
public PartialListing list(String prefix, int maxListingLength,
String priorLastKey) throws IOException {
+
+ return list(prefix, PATH_DELIMITER, maxListingLength, priorLastKey);
+ }
+
+ public PartialListing listAll(String prefix, int maxListingLength,
+ String priorLastKey) throws IOException {
+
+ return list(prefix, null, maxListingLength, priorLastKey);
+ }
+
+ private PartialListing list(String prefix, String delimiter,
+ int maxListingLength, String priorLastKey) throws IOException {
try {
- if (prefix.length() > 0 &&
- !prefix.endsWith(NativeS3FileSystem.PATH_DELIMITER)) {
- prefix += NativeS3FileSystem.PATH_DELIMITER;
+ if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
}
S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
- prefix, NativeS3FileSystem.PATH_DELIMITER, maxListingLength,
- priorLastKey);
+ prefix, delimiter, maxListingLength, priorLastKey);
FileMetadata[] fileMetadata =
new FileMetadata[chunk.getObjects().length];
@@ -210,6 +222,18 @@
throw new S3Exception(e);
}
}
+
+ public void rename(String srcKey, String dstKey) throws IOException {
+ try {
+ s3Service.moveObject(bucket.getName(), srcKey, bucket.getName(),
+ new S3Object(dstKey), false);
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
public void purge(String prefix) throws IOException {
try {
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
---
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
(original)
+++
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
Mon Sep 8 03:47:40 2008
@@ -44,8 +44,12 @@
PartialListing list(String prefix, int maxListingLength) throws IOException;
PartialListing list(String prefix, int maxListingLength, String priorLastKey)
throws IOException;
+ PartialListing listAll(String prefix, int maxListingLength,
+ String priorLastKey) throws IOException;
void delete(String key) throws IOException;
+
+ void rename(String srcKey, String dstKey) throws IOException;
/**
* Delete all keys with the given prefix. Used for testing.
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
---
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
(original)
+++
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
Mon Sep 8 03:47:40 2008
@@ -451,12 +451,116 @@
return new FSDataInputStream(new BufferedFSInputStream(
new NativeS3FsInputStream(store.retrieve(key), key), bufferSize));
}
+
+ // rename() and delete() use this method to ensure that the parent directory
+ // of the source does not vanish.
+ private void createParent(Path path) throws IOException {
+ Path parent = path.getParent();
+ if (parent != null) {
+ String key = pathToKey(makeAbsolute(parent));
+ if (key.length() > 0) {
+ store.storeEmptyFile(key + FOLDER_SUFFIX);
+ }
+ }
+ }
+
+ private boolean existsAndIsFile(Path f) throws IOException {
+
+ Path absolutePath = makeAbsolute(f);
+ String key = pathToKey(absolutePath);
+
+ if (key.length() == 0) {
+ return false;
+ }
+
+ FileMetadata meta = store.retrieveMetadata(key);
+ if (meta != null) {
+ // S3 object with given key exists, so this is a file
+ return true;
+ }
+
+ if (store.retrieveMetadata(key + FOLDER_SUFFIX) != null) {
+ // Signifies empty directory
+ return false;
+ }
+
+ PartialListing listing = store.list(key, 1, null);
+ if (listing.getFiles().length > 0 ||
+ listing.getCommonPrefixes().length > 0) {
+ // Non-empty directory
+ return false;
+ }
+
+ throw new FileNotFoundException(absolutePath +
+ ": No such file or directory");
+}
+
@Override
public boolean rename(Path src, Path dst) throws IOException {
- throw new IOException("Not supported");
+
+ String srcKey = pathToKey(makeAbsolute(src));
+
+ if (srcKey.length() == 0) {
+ // Cannot rename root of file system
+ return false;
+ }
+
+ // Figure out the final destination
+ String dstKey;
+ try {
+ boolean dstIsFile = existsAndIsFile(dst);
+ if (dstIsFile) {
+ // Attempting to overwrite a file using rename()
+ return false;
+ } else {
+ // Move to within the existent directory
+ dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
+ }
+ } catch (FileNotFoundException e) {
+ // dst doesn't exist, so we can proceed
+ dstKey = pathToKey(makeAbsolute(dst));
+ try {
+ if (!getFileStatus(dst.getParent()).isDir()) {
+ return false; // parent dst is a file
+ }
+ } catch (FileNotFoundException ex) {
+ return false; // parent dst does not exist
+ }
+ }
+
+ try {
+ boolean srcIsFile = existsAndIsFile(src);
+ if (srcIsFile) {
+ store.rename(srcKey, dstKey);
+ } else {
+ // Move the folder object
+ store.delete(srcKey + FOLDER_SUFFIX);
+ store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
+
+ // Move everything inside the folder
+ String priorLastKey = null;
+ do {
+ PartialListing listing = store.listAll(srcKey, S3_MAX_LISTING_LENGTH,
+ priorLastKey);
+ for (FileMetadata file : listing.getFiles()) {
+ store.rename(file.getKey(), dstKey
+ + file.getKey().substring(srcKey.length()));
+ }
+ priorLastKey = listing.getPriorLastKey();
+ } while (priorLastKey != null);
+ }
+
+ createParent(src);
+ return true;
+
+ } catch (FileNotFoundException e) {
+ // Source file does not exist;
+ return false;
+ }
}
+
/**
* Set the working directory to the given directory.
*/
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
(original)
+++
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
Mon Sep 8 03:47:40 2008
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3native;
+import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
@@ -127,22 +128,36 @@
public PartialListing list(String prefix, int maxListingLength,
String priorLastKey) throws IOException {
- if (prefix.length() > 0 &&
- !prefix.endsWith(NativeS3FileSystem.PATH_DELIMITER)) {
- prefix += NativeS3FileSystem.PATH_DELIMITER;
+ return list(prefix, PATH_DELIMITER, maxListingLength, priorLastKey);
+ }
+
+ public PartialListing listAll(String prefix, int maxListingLength,
+ String priorLastKey) throws IOException {
+
+ return list(prefix, null, maxListingLength, priorLastKey);
+ }
+
+ private PartialListing list(String prefix, String delimiter,
+ int maxListingLength, String priorLastKey) throws IOException {
+
+ if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
}
List<FileMetadata> metadata = new ArrayList<FileMetadata>();
SortedSet<String> commonPrefixes = new TreeSet<String>();
for (String key : dataMap.keySet()) {
if (key.startsWith(prefix)) {
- int delimIndex = key.indexOf(NativeS3FileSystem.PATH_DELIMITER,
- prefix.length());
- if (delimIndex == -1) {
+ if (delimiter == null) {
metadata.add(retrieveMetadata(key));
} else {
- String commonPrefix = key.substring(0, delimIndex);
- commonPrefixes.add(commonPrefix);
+ int delimIndex = key.indexOf(delimiter, prefix.length());
+ if (delimIndex == -1) {
+ metadata.add(retrieveMetadata(key));
+ } else {
+ String commonPrefix = key.substring(0, delimIndex);
+ commonPrefixes.add(commonPrefix);
+ }
}
}
if (metadata.size() + commonPrefixes.size() == maxListingLength) {
@@ -159,6 +174,11 @@
dataMap.remove(key);
}
+ public void rename(String srcKey, String dstKey) throws IOException {
+ metadataMap.put(dstKey, metadataMap.remove(srcKey));
+ dataMap.put(dstKey, dataMap.remove(srcKey));
+ }
+
public void purge(String prefix) throws IOException {
Iterator<Entry<String, FileMetadata>> i =
metadataMap.entrySet().iterator();
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
(original)
+++
hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
Mon Sep 8 03:47:40 2008
@@ -47,11 +47,6 @@
super.tearDown();
}
- @Override
- protected boolean renameSupported() {
- return false;
- }
-
public void testListStatusForRoot() throws Exception {
Path testDir = path("/test");
assertTrue(fs.mkdirs(testDir));
