Author: szetszwo Date: Tue May 1 01:08:39 2012 New Revision: 1332486 URL: http://svn.apache.org/viewvc?rev=1332486&view=rev Log: svn merge -c 932042 from branch-0.20 for HDFS-1041.
Modified: hadoop/common/branches/branch-1.0/ (props changed) hadoop/common/branches/branch-1.0/CHANGES.txt (contents, props changed) hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java Propchange: hadoop/common/branches/branch-1.0/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-0.20:r932042 Modified: hadoop/common/branches/branch-1.0/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1332486&r1=1332485&r2=1332486&view=diff ============================================================================== --- hadoop/common/branches/branch-1.0/CHANGES.txt (original) +++ hadoop/common/branches/branch-1.0/CHANGES.txt Tue May 1 01:08:39 2012 @@ -55,6 +55,9 @@ Release 1.0.3 - unreleased HDFS-3316. The tar ball doesn't include jsvc any more. (Owen O'Malley via mattf) + HDFS-1041. DFSClient.getFileChecksum(..) should retry if connection to + the first datanode fails. (szetszwo) + Release 1.0.2 - 2012.03.24 NEW FEATURES Propchange: hadoop/common/branches/branch-1.0/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-0.20/CHANGES.txt:r932042 Modified: hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=1332486&r1=1332485&r2=1332486&view=diff ============================================================================== --- hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Tue May 1 01:08:39 2012 @@ -532,7 +532,7 @@ public class DFSClient implements FSCons return hints; } - private static LocatedBlocks callGetBlockLocations(ClientProtocol namenode, + static LocatedBlocks callGetBlockLocations(ClientProtocol namenode, String src, long start, long length) throws IOException { try { return namenode.getBlockLocations(src, start, length); @@ -929,25 +929,28 @@ public class DFSClient implements FSCons boolean done = false; for(int j = 0; !done && j < datanodes.length; j++) { - //connect to a datanode - final Socket sock = socketFactory.createSocket(); - NetUtils.connect(sock, - NetUtils.createSocketAddr(datanodes[j].getName()), - timeout); - sock.setSoTimeout(timeout); - - DataOutputStream out = new DataOutputStream( - new BufferedOutputStream(NetUtils.getOutputStream(sock), - DataNode.SMALL_BUFFER_SIZE)); - DataInputStream in = new DataInputStream(NetUtils.getInputStream(sock)); - - // get block MD5 + Socket sock = null; + DataOutputStream out = null; + DataInputStream in = null; + try { + //connect to a datanode + sock = socketFactory.createSocket(); + NetUtils.connect(sock, + NetUtils.createSocketAddr(datanodes[j].getName()), timeout); + sock.setSoTimeout(timeout); + + out = new DataOutputStream( + new BufferedOutputStream(NetUtils.getOutputStream(sock), + DataNode.SMALL_BUFFER_SIZE)); + in = new DataInputStream(NetUtils.getInputStream(sock)); + if (LOG.isDebugEnabled()) { LOG.debug("write to " + datanodes[j].getName() + ": " - + DataTransferProtocol.OP_BLOCK_CHECKSUM + - ", block=" + block); + + DataTransferProtocol.OP_BLOCK_CHECKSUM + ", block=" + block); } + + // get block MD5 out.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION); out.write(DataTransferProtocol.OP_BLOCK_CHECKSUM); out.writeLong(block.getBlockId()); Modified: hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1332486&r1=1332485&r2=1332486&view=diff ============================================================================== --- hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original) +++ hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java Tue May 1 01:08:39 2012 @@ -478,4 +478,35 @@ public class TestDFSClientRetries extend server.stop(); } } + + public void testGetFileChecksum() throws Exception { + final String f = "/testGetFileChecksum"; + final Path p = new Path(f); + + final Configuration conf = new Configuration(); + final MiniDFSCluster cluster = new MiniDFSCluster(conf, 3, true, null); + try { + cluster.waitActive(); + + //create a file + final FileSystem fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, p, 1L << 20, (short)3, 20100402L); + + //get checksum + final FileChecksum cs1 = fs.getFileChecksum(p); + assertTrue(cs1 != null); + + //stop the first datanode + final List<LocatedBlock> locatedblocks = DFSClient.callGetBlockLocations( + cluster.getNameNode(), f, 0, Long.MAX_VALUE).getLocatedBlocks(); + final DatanodeInfo first = locatedblocks.get(0).getLocations()[0]; + cluster.stopDataNode(first.getName()); + + //get checksum again + final FileChecksum cs2 = fs.getFileChecksum(p); + assertEquals(cs1, cs2); + } finally { + cluster.shutdown(); + } + } }