svn commit: r781343 - in /hadoop/core/trunk: ./ src/core/ src/core/org/apache/hadoop/fs/s3/ src/core/org/apache/hadoop/fs/s3native/ src/test/core/org/apache/hadoop/fs/s3/ src/test/core/org/apache/hadoop/fs/s3native/

[tomwhite]

Author: tomwhite
Date: Wed Jun  3 11:28:03 2009
New Revision: 781343

URL: http://svn.apache.org/viewvc?rev=781343&view=rev
Log:
HADOOP-5861. s3n files are not getting split by default.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/core-default.xml
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
    
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
    
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
    
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=781343&r1=781342&r2=781343&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jun  3 11:28:03 2009
@@ -772,6 +772,8 @@
     HADOOP-5836. Bug in S3N handling of directory markers using an object with
     a trailing "/" causes jobs to fail. (Ian Nowland via tomwhite)
 
+    HADOOP-5861. s3n files are not getting split by default. (tomwhite)
+
 Release 0.20.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/core/core-default.xml
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/core-default.xml?rev=781343&r1=781342&r2=781343&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/core-default.xml (original)
+++ hadoop/core/trunk/src/core/core-default.xml Wed Jun  3 11:28:03 2009
@@ -257,6 +257,12 @@
   </description>
 </property>
 
+<property>
+  <name>fs.s3n.block.size</name>
+  <value>67108864</value>
+  <description>Block size to use when reading files using the native S3
+  filesystem (s3n: URIs).</description>
+</property>
 
 <property>
   <name>local.cache.size</name>

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=781343&r1=781342&r2=781343&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java 
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java Wed 
Jun  3 11:28:03 2009
@@ -332,6 +332,11 @@
     }
     return new S3FileStatus(f.makeQualified(this), inode);
   }
+  
+  @Override
+  public long getDefaultBlockSize() {
+    return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024);
+  }
 
   // diagnostic methods
 

Modified: 
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=781343&r1=781342&r2=781343&view=diff
==============================================================================
--- 
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
 (original)
+++ 
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
 Wed Jun  3 11:28:03 2009
@@ -83,7 +83,6 @@
     LogFactory.getLog(NativeS3FileSystem.class);
   
   private static final String FOLDER_SUFFIX = "_$folder$";
-  private static final long MAX_S3_FILE_SIZE = 5 * 1024 * 1024 * 1024L;
   static final String PATH_DELIMITER = Path.SEPARATOR;
   private static final int S3_MAX_LISTING_LENGTH = 1000;
   
@@ -464,13 +463,12 @@
   }
   
   private FileStatus newFile(FileMetadata meta, Path path) {
-    return new FileStatus(meta.getLength(), false, 1, MAX_S3_FILE_SIZE,
+    return new FileStatus(meta.getLength(), false, 1, getDefaultBlockSize(),
         meta.getLastModified(), path.makeQualified(this));
   }
   
   private FileStatus newDirectory(Path path) {
-    return new FileStatus(0, true, 1, MAX_S3_FILE_SIZE, 0,
-        path.makeQualified(this));
+    return new FileStatus(0, true, 1, 0, 0, path.makeQualified(this));
   }
 
   @Override
@@ -609,6 +607,11 @@
 
     return true;
   }
+  
+  @Override
+  public long getDefaultBlockSize() {
+    return getConf().getLong("fs.s3n.block.size", 64 * 1024 * 1024);
+  }
 
   /**
    * Set the working directory to the given directory.

Modified: 
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java?rev=781343&r1=781342&r2=781343&view=diff
==============================================================================
--- 
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
 (original)
+++ 
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
 Wed Jun  3 11:28:03 2009
@@ -23,6 +23,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
 
 public abstract class S3FileSystemContractBaseTest
   extends FileSystemContractBaseTest {
@@ -45,4 +46,15 @@
     super.tearDown();
   }
   
+  public void testBlockSize() throws Exception {
+    
+    long newBlockSize = fs.getDefaultBlockSize() * 2;
+    fs.getConf().setLong("fs.s3.block.size", newBlockSize);
+    
+    Path file = path("/test/hadoop/file");
+    createFile(file);
+    assertEquals("Double default block size", newBlockSize,
+       fs.getFileStatus(file).getBlockSize());
+  }
+  
 }

Modified: 
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java?rev=781343&r1=781342&r2=781343&view=diff
==============================================================================
--- 
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
 (original)
+++ 
hadoop/core/trunk/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
 Wed Jun  3 11:28:03 2009
@@ -132,4 +132,18 @@
     store.storeEmptyFile("test/hadoop/file1");
     fs.open(path("/test/hadoop/file1")).close();
   }
+  
+  public void testBlockSize() throws Exception {
+    Path file = path("/test/hadoop/file");
+    createFile(file);
+    assertEquals("Default block size", fs.getDefaultBlockSize(),
+    fs.getFileStatus(file).getBlockSize());
+
+    // Block size is determined at read time
+    long newBlockSize = fs.getDefaultBlockSize() * 2;
+    fs.getConf().setLong("fs.s3n.block.size", newBlockSize);
+    assertEquals("Double default block size", newBlockSize,
+    fs.getFileStatus(file).getBlockSize());
+  }
+
 }