svn commit: r1043308 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java

[cos]

Author: cos
Date: Wed Dec  8 05:44:38 2010
New Revision: 1043308

URL: http://svn.apache.org/viewvc?rev=1043308&view=rev
Log:
HDFS-1527. SocketOutputStream.transferToFully fails for blocks >= 2GB on 32 bit 
JVM. Contributed by Patrick Kling.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1043308&r1=1043307&r2=1043308&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Wed Dec  8 05:44:38 2010
@@ -431,6 +431,9 @@ Release 0.22.0 - Unreleased
     HDFS-1532. Exclude Findbugs warning in FSImageFormat$Saver. (Todd Lipcon
     via cos)
 
+    HDFS-1527. SocketOutputStream.transferToFully fails for blocks >= 2GB on
+    32 bit JVM. (Patrick Kling via cos)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: 
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
URL: 
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java?rev=1043308&r1=1043307&r2=1043308&view=diff
==============================================================================
--- 
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
 (original)
+++ 
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
 Wed Dec  8 05:44:38 2010
@@ -54,6 +54,8 @@ class BlockSender implements java.io.Clo
   /** The visible length of a replica. */
   private final long replicaVisibleLength;
 
+  private static final boolean is32Bit = 
System.getProperty("sun.arch.data.model").equals("32");
+
   private InputStream blockIn; // data stream
   private long blockInPosition = -1; // updated while using transferTo().
   private DataInputStream checksumIn; // checksum datastream
@@ -143,7 +145,11 @@ class BlockSender implements java.io.Clo
       this.chunkOffsetOK = chunkOffsetOK;
       this.corruptChecksumOk = corruptChecksumOk;
       this.verifyChecksum = verifyChecksum;
-      this.transferToAllowed = datanode.transferToAllowed;
+
+      // transferToFully() fails on 32 bit platforms for block sizes >= 2GB,
+      // use normal transfer in those cases
+      this.transferToAllowed = datanode.transferToAllowed &&
+        (!is32Bit || length < (long) Integer.MAX_VALUE);
       this.clientTraceFmt = clientTraceFmt;
 
       if ( !corruptChecksumOk || datanode.data.metaFileExists(block) ) {
@@ -396,6 +402,7 @@ class BlockSender implements java.io.Clo
       /* exception while writing to the client (well, with transferTo(),
        * it could also be while reading from the local file).
        */
+      LOG.error("BlockSender.sendChunks() exception: " + 
StringUtils.stringifyException(e));
       throw ioeToSocketException(e);
     }
 

Modified: 
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
URL: 
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java?rev=1043308&r1=1043307&r2=1043308&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java 
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java 
Wed Dec  8 05:44:38 2010
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdfs.server.nam
 
 import org.junit.Test;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
 
 /**
  * This class tests that blocks can be larger than 2GB
@@ -58,7 +57,6 @@ public class TestLargeBlock {
   static final boolean verifyData = true;
   static final byte[] pattern = { 'D', 'E', 'A', 'D', 'B', 'E', 'E', 'F'};
   static final boolean simulatedStorage = false;
-  private static final String ALLOWED_VM = "64";
 
   // creates a file 
   static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl,
@@ -167,7 +165,6 @@ public class TestLargeBlock {
    */
   @Test
   public void testLargeBlockSize() throws IOException {
-    assumeTrue(ALLOWED_VM.equals(System.getProperty("sun.arch.data.model")));
     final long blockSize = 2L * 1024L * 1024L * 1024L + 512L; // 2GB + 512B
     runTest(blockSize);
   }