Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java Fri Sep 3 23:38:21 2010 @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FSDataOutput import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; @@ -158,7 +159,7 @@ public class TestFileAppend4 { // Delay completeFile DelayAnswer delayer = new DelayAnswer(); doAnswer(delayer).when(spyNN).complete( - anyString(), anyString(), (Block)anyObject()); + anyString(), anyString(), (ExtendedBlock)anyObject()); DFSClient client = new DFSClient(null, spyNN, conf, null); file1 = new Path("/testRecoverFinalized"); @@ -228,7 +229,8 @@ public class TestFileAppend4 { // Delay completeFile DelayAnswer delayer = new DelayAnswer(); - doAnswer(delayer).when(spyNN).complete(anyString(), anyString(), (Block)anyObject()); + doAnswer(delayer).when(spyNN).complete(anyString(), anyString(), + (ExtendedBlock) anyObject()); DFSClient client = new DFSClient(null, spyNN, conf, null); file1 = new Path("/testCompleteOtherLease");
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java Fri Sep 3 23:38:21 2010 @@ -38,9 +38,9 @@ import org.apache.hadoop.fs.ChecksumExce import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -131,7 +131,7 @@ public class TestFileCorruption extends // get the block File dataDir = new File(cluster.getDataDirectory(), "data1" + MiniDFSCluster.FINALIZED_DIR_NAME); - Block blk = getBlock(dataDir); + ExtendedBlock blk = getBlock(dataDir); if (blk == null) { blk = getBlock(new File(cluster.getDataDirectory(), "dfs/data/data2" + MiniDFSCluster.FINALIZED_DIR_NAME)); @@ -159,7 +159,7 @@ public class TestFileCorruption extends } - private Block getBlock(File dataDir) { + private ExtendedBlock getBlock(File dataDir) { assertTrue("data directory does not exist", dataDir.exists()); File[] blocks = dataDir.listFiles(); assertTrue("Blocks do not exist in dataDir", (blocks != null) && (blocks.length > 0)); @@ -186,7 +186,8 @@ public class TestFileCorruption extends break; } } - return new Block(blockId, blocks[idx].length(), blockTimeStamp); + // TODO:FEDERATION cleanup when BlockPoolID support in Datanode is complete + return new ExtendedBlock("TODO", blockId, blocks[idx].length(), blockTimeStamp); } /** check if ClientProtocol.getCorruptFiles() returns a file that has missing blocks */ Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java Fri Sep 3 23:38:21 2010 @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -34,7 +35,7 @@ public class TestLeaseRecovery extends j static final short REPLICATION_NUM = (short)3; private static final long LEASE_PERIOD = 300L; - static void checkMetaInfo(Block b, DataNode dn + static void checkMetaInfo(ExtendedBlock b, DataNode dn ) throws IOException { TestInterDatanodeProtocol.checkMetaInfo(b, dn); } @@ -96,7 +97,7 @@ public class TestLeaseRecovery extends j } //verify Block Info - Block lastblock = locatedblock.getBlock(); + ExtendedBlock lastblock = locatedblock.getBlock(); DataNode.LOG.info("newblocks=" + lastblock); for(int i = 0; i < REPLICATION_NUM; i++) { checkMetaInfo(lastblock, datanodes[i]); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java Fri Sep 3 23:38:21 2010 @@ -66,8 +66,8 @@ public class TestMissingBlocksAlert exte // Corrupt the block - String block = DFSTestUtil.getFirstBlock(dfs, corruptFile).getBlockName(); - TestDatanodeBlockScanner.corruptReplica(block, 0); + String block = DFSTestUtil.getFirstBlock(dfs, corruptFile).getLocalBlock().getBlockName(); + assertTrue(TestDatanodeBlockScanner.corruptReplica(block, 0)); // read the file so that the corrupt block is reported to NN FSDataInputStream in = dfs.open(corruptFile); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Fri Sep 3 23:38:21 2010 @@ -32,6 +32,7 @@ import org.apache.commons.logging.impl.L import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.io.TestWritable; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.RPC; @@ -81,9 +82,9 @@ public class TestBlockToken { long blockKeyUpdateInterval = 10 * 60 * 1000; // 10 mins long blockTokenLifetime = 2 * 60 * 1000; // 2 mins - Block block1 = new Block(0L); - Block block2 = new Block(10L); - Block block3 = new Block(-108L); + ExtendedBlock block1 = new ExtendedBlock("0", 0L); + ExtendedBlock block2 = new ExtendedBlock("10", 10L); + ExtendedBlock block3 = new ExtendedBlock("-10", -108L); private static class getLengthAnswer implements Answer<Long> { BlockTokenSecretManager sm; @@ -99,7 +100,7 @@ public class TestBlockToken { public Long answer(InvocationOnMock invocation) throws IOException { Object args[] = invocation.getArguments(); assertEquals(1, args.length); - Block block = (Block) args[0]; + ExtendedBlock block = (ExtendedBlock) args[0]; Set<TokenIdentifier> tokenIds = UserGroupInformation.getCurrentUser() .getTokenIdentifiers(); assertEquals("Only one BlockTokenIdentifier expected", 1, tokenIds.size()); @@ -117,7 +118,7 @@ public class TestBlockToken { } private BlockTokenIdentifier generateTokenId(BlockTokenSecretManager sm, - Block block, EnumSet<BlockTokenSecretManager.AccessMode> accessModes) + ExtendedBlock block, EnumSet<BlockTokenSecretManager.AccessMode> accessModes) throws IOException { Token<BlockTokenIdentifier> token = sm.generateToken(block, accessModes); BlockTokenIdentifier id = sm.createIdentifier(); @@ -197,7 +198,7 @@ public class TestBlockToken { id.readFields(new DataInputStream(new ByteArrayInputStream(token .getIdentifier()))); doAnswer(new getLengthAnswer(sm, id)).when(mockDN).getReplicaVisibleLength( - any(Block.class)); + any(ExtendedBlock.class)); final Server server = RPC.getServer(ClientDatanodeProtocol.class, mockDN, ADDRESS, 0, 5, true, conf, sm); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java Fri Sep 3 23:38:21 2010 @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; @@ -82,7 +83,8 @@ public class TestBalancer extends TestCa /* fill up a cluster with <code>numNodes</code> datanodes * whose used space to be <code>size</code> */ - private Block[] generateBlocks(Configuration conf, long size, short numNodes) throws IOException { + private ExtendedBlock[] generateBlocks(Configuration conf, long size, + short numNodes) throws IOException { cluster = new MiniDFSCluster( conf, numNodes, true, null); try { cluster.waitActive(); @@ -96,10 +98,11 @@ public class TestBalancer extends TestCa getBlockLocations(fileName, 0, fileLen).getLocatedBlocks(); int numOfBlocks = locatedBlocks.size(); - Block[] blocks = new Block[numOfBlocks]; + ExtendedBlock[] blocks = new ExtendedBlock[numOfBlocks]; for(int i=0; i<numOfBlocks; i++) { - Block b = locatedBlocks.get(i).getBlock(); - blocks[i] = new Block(b.getBlockId(), b.getNumBytes(), b.getGenerationStamp()); + ExtendedBlock b = locatedBlocks.get(i).getBlock(); + blocks[i] = new ExtendedBlock(b.getPoolId(), b.getBlockId(), b + .getNumBytes(), b.getGenerationStamp()); } return blocks; @@ -109,7 +112,7 @@ public class TestBalancer extends TestCa } /* Distribute all blocks according to the given distribution */ - Block[][] distributeBlocks(Block[] blocks, short replicationFactor, + Block[][] distributeBlocks(ExtendedBlock[] blocks, short replicationFactor, final long[] distribution ) { // make a copy long[] usedSpace = new long[distribution.length]; @@ -128,7 +131,7 @@ public class TestBalancer extends TestCa int chosenIndex = r.nextInt(usedSpace.length); if( usedSpace[chosenIndex]>0 ) { notChosen = false; - blockReports.get(chosenIndex).add(blocks[i]); + blockReports.get(chosenIndex).add(blocks[i].getLocalBlock()); usedSpace[chosenIndex] -= blocks[i].getNumBytes(); } } @@ -159,7 +162,8 @@ public class TestBalancer extends TestCa } // fill the cluster - Block[] blocks = generateBlocks(conf, totalUsedSpace, (short)numDatanodes); + ExtendedBlock[] blocks = generateBlocks(conf, totalUsedSpace, + (short) numDatanodes); // redistribute blocks Block[][] blocksDN = distributeBlocks( Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Fri Sep 3 23:38:21 2010 @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; @@ -793,8 +794,8 @@ public class SimulatedFSDataset impleme @Override public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) throws IOException { - Block b = rBlock.getBlock(); - BInfo binfo = blockMap.get(b); + ExtendedBlock b = rBlock.getBlock(); + BInfo binfo = blockMap.get(b.getLocalBlock()); if (binfo == null) { throw new IOException("No such Block " + b ); } Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Fri Sep 3 23:38:21 2010 @@ -41,9 +41,9 @@ import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DataTransferProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; @@ -118,7 +118,7 @@ public class TestBlockReplacement extend LocatedBlock block = locatedBlocks.get(0); DatanodeInfo[] oldNodes = block.getLocations(); assertEquals(oldNodes.length, 3); - Block b = block.getBlock(); + ExtendedBlock b = block.getBlock(); // add a new datanode to the cluster cluster.startDataNodes(CONF, 1, true, null, NEW_RACKS); @@ -222,7 +222,7 @@ public class TestBlockReplacement extend * * Return true if a block is successfully copied; otherwise false. */ - private boolean replaceBlock( Block block, DatanodeInfo source, + private boolean replaceBlock( ExtendedBlock block, DatanodeInfo source, DatanodeInfo sourceProxy, DatanodeInfo destination) throws IOException { Socket sock = new Socket(); sock.connect(NetUtils.createSocketAddr( Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Fri Sep 3 23:38:21 2010 @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.Distribute import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.common.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -132,6 +133,7 @@ public class TestBlockReport { } cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N0).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); List<LocatedBlock> blocksAfterReport = @@ -143,7 +145,7 @@ public class TestBlockReport { } for (int i = 0; i < blocksAfterReport.size(); i++) { - Block b = blocksAfterReport.get(i).getBlock(); + ExtendedBlock b = blocksAfterReport.get(i).getBlock(); assertEquals("Length of " + i + "th block is incorrect", oldLengths[i], b.getNumBytes()); } @@ -184,7 +186,7 @@ public class TestBlockReport { } for (Integer aRemovedIndex : removedIndex) { - blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock()); + blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock().getLocalBlock()); } ArrayList<Block> blocks = locatedToBlocks(lBlocks, removedIndex); @@ -208,6 +210,7 @@ public class TestBlockReport { cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N0).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); cluster.getNamesystem().computeDatanodeWork(); @@ -244,6 +247,7 @@ public class TestBlockReport { DatanodeCommand dnCmd = cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N0).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); if(LOG.isDebugEnabled()) { LOG.debug("Got the command: " + dnCmd); @@ -293,6 +297,7 @@ public class TestBlockReport { cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N1).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); printStats(); assertEquals("Wrong number of PendingReplication Blocks", @@ -340,6 +345,7 @@ public class TestBlockReport { } cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N1).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); printStats(); assertEquals("Wrong number of Corrupted blocks", @@ -362,6 +368,7 @@ public class TestBlockReport { } cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N1).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); printStats(); @@ -409,6 +416,7 @@ public class TestBlockReport { cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N1).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -452,6 +460,7 @@ public class TestBlockReport { cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N1).dnRegistration, + cluster.getNamesystem().getPoolId(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -593,7 +602,7 @@ public class TestBlockReport { } continue; } - newList.add(new Block(locatedBlks.get(i).getBlock())); + newList.add(new Block(locatedBlks.get(i).getBlock().getLocalBlock())); } return newList; } Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Fri Sep 3 23:38:21 2010 @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.common.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -119,7 +120,8 @@ public class TestDataNodeVolumeFailure e // make sure a block report is sent DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3 long[] bReport = dn.getFSDataset().getBlockReport().getBlockListAsLongs(); - cluster.getNameNode().blockReport(dn.dnRegistration, bReport); + String poolId = cluster.getNamesystem().getPoolId(); + cluster.getNameNode().blockReport(dn.dnRegistration, poolId, bReport); // verify number of blocks and files... verify(filename, filesize); @@ -197,7 +199,7 @@ public class TestDataNodeVolumeFailure e for(LocatedBlock lb : locatedBlocks) { DatanodeInfo dinfo = lb.getLocations()[1]; - Block b = lb.getBlock(); + ExtendedBlock b = lb.getBlock(); // System.out.println(i++ + ". " + b.getBlockName()); try { accessBlock(dinfo, lb); @@ -238,8 +240,7 @@ public class TestDataNodeVolumeFailure e throws IOException { InetSocketAddress targetAddr = null; Socket s = null; - BlockReader blockReader = null; - Block block = lblock.getBlock(); + ExtendedBlock block = lblock.getBlock(); targetAddr = NetUtils.createSocketAddr(datanode.getName()); @@ -247,7 +248,7 @@ public class TestDataNodeVolumeFailure e s.connect(targetAddr, HdfsConstants.READ_TIMEOUT); s.setSoTimeout(HdfsConstants.READ_TIMEOUT); - blockReader = + BlockReader blockReader = BlockReader.newBlockReader(s, targetAddr.toString() + ":" + block.getBlockId(), block.getBlockId(), Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java Fri Sep 3 23:38:21 2010 @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; @@ -46,7 +47,7 @@ import org.junit.Test; * This tests InterDataNodeProtocol for block handling. */ public class TestInterDatanodeProtocol { - public static void checkMetaInfo(Block b, DataNode dn) throws IOException { + public static void checkMetaInfo(ExtendedBlock b, DataNode dn) throws IOException { Block metainfo = dn.data.getStoredBlock(b.getBlockId()); Assert.assertEquals(b.getBlockId(), metainfo.getBlockId()); Assert.assertEquals(b.getNumBytes(), metainfo.getNumBytes()); @@ -100,7 +101,7 @@ public class TestInterDatanodeProtocol { datanode.blockScannerThread.interrupt(); //verify BlockMetaDataInfo - Block b = locatedblock.getBlock(); + ExtendedBlock b = locatedblock.getBlock(); InterDatanodeProtocol.LOG.info("b=" + b + ", " + b.getClass()); checkMetaInfo(b, datanode); long recoveryId = b.getGenerationStamp() + 1; @@ -108,7 +109,7 @@ public class TestInterDatanodeProtocol { new RecoveringBlock(b, locatedblock.getLocations(), recoveryId)); //verify updateBlock - Block newblock = new Block( + ExtendedBlock newblock = new ExtendedBlock(b.getPoolId(), b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1); idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes()); checkMetaInfo(newblock, datanode); @@ -208,7 +209,7 @@ public class TestInterDatanodeProtocol { } } - /** Test {...@link FSDataset#updateReplicaUnderRecovery(ReplicaUnderRecovery, long, long)} */ + /** Test {...@link FSDataset#updateReplicaUnderRecovery(Block, long, long)} */ @Test public void testUpdateReplicaUnderRecovery() throws IOException { final Configuration conf = new HdfsConfiguration(); @@ -237,7 +238,7 @@ public class TestInterDatanodeProtocol { final FSDataset fsdataset = (FSDataset)datanode.data; //initReplicaRecovery - final Block b = locatedblock.getBlock(); + final ExtendedBlock b = locatedblock.getBlock(); final long recoveryid = b.getGenerationStamp() + 1; final long newlength = b.getNumBytes() - 1; final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery( Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Fri Sep 3 23:38:21 2010 @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -73,7 +74,7 @@ import org.apache.log4j.LogManager; * <li>-logLevel L specifies the logging level when the benchmark runs. * The default logging level is {...@link Level#ERROR}.</li> * <li>-UGCacheRefreshCount G will cause the benchmark to call - * {...@link NameNode#refreshUserToGroupsMappings(Configuration)} after + * {...@link NameNode#refreshUserToGroupsMappings()} after * every G operations, which purges the name-node's user group cache. * By default the refresh is never called.</li> * </ol> @@ -842,6 +843,7 @@ public class NNThroughputBenchmark { new DataStorage(nsInfo, dnInfo.getStorageID())); receivedDNReg.setInfoPort(dnInfo.getInfoPort()); nameNode.blockReceived( receivedDNReg, + nameNode.getNamesystem().getPoolId(), new Block[] {blocks[i]}, new String[] {DataNode.EMPTY_DEL_HINT}); } @@ -937,7 +939,7 @@ public class NNThroughputBenchmark { nameNode.create(fileName, FsPermission.getDefault(), clientName, new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.OVERWRITE)), true, replication, BLOCK_SIZE); - Block lastBlock = addBlocks(fileName, clientName); + ExtendedBlock lastBlock = addBlocks(fileName, clientName); nameNode.complete(fileName, clientName, lastBlock); } // prepare block reports @@ -946,18 +948,19 @@ public class NNThroughputBenchmark { } } - private Block addBlocks(String fileName, String clientName) + private ExtendedBlock addBlocks(String fileName, String clientName) throws IOException { - Block prevBlock = null; + ExtendedBlock prevBlock = null; for(int jdx = 0; jdx < blocksPerFile; jdx++) { LocatedBlock loc = nameNode.addBlock(fileName, clientName, prevBlock, null); prevBlock = loc.getBlock(); for(DatanodeInfo dnInfo : loc.getLocations()) { int dnIdx = Arrays.binarySearch(datanodes, dnInfo.getName()); - datanodes[dnIdx].addBlock(loc.getBlock()); + datanodes[dnIdx].addBlock(loc.getBlock().getLocalBlock()); nameNode.blockReceived( datanodes[dnIdx].dnRegistration, - new Block[] {loc.getBlock()}, + loc.getBlock().getPoolId(), + new Block[] {loc.getBlock().getLocalBlock()}, new String[] {""}); } } @@ -975,7 +978,8 @@ public class NNThroughputBenchmark { assert daemonId < numThreads : "Wrong daemonId."; TinyDatanode dn = datanodes[daemonId]; long start = System.currentTimeMillis(); - nameNode.blockReport(dn.dnRegistration, dn.getBlockReportList()); + nameNode.blockReport(dn.dnRegistration, nameNode.getNamesystem() + .getPoolId(), dn.getBlockReportList()); long end = System.currentTimeMillis(); return end-start; } Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java Fri Sep 3 23:38:21 2010 @@ -31,8 +31,8 @@ import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.security.token.block.*; import org.apache.hadoop.hdfs.server.balancer.TestBalancer; @@ -122,7 +122,7 @@ public class TestBlockTokenWithDFS exten InetSocketAddress targetAddr = null; Socket s = null; BlockReader blockReader = null; - Block block = lblock.getBlock(); + ExtendedBlock block = lblock.getBlock(); try { DatanodeInfo[] nodes = lblock.getLocations(); targetAddr = NetUtils.createSocketAddr(nodes[0].getName()); @@ -352,7 +352,8 @@ public class TestBlockTokenWithDFS exten // read should succeed tryRead(conf, lblock, true); // use a token with wrong blockID - Block wrongBlock = new Block(lblock.getBlock().getBlockId() + 1); + ExtendedBlock wrongBlock = new ExtendedBlock(lblock.getBlock() + .getPoolId(), lblock.getBlock().getBlockId() + 1); lblock.setBlockToken(cluster.getNameNode().getNamesystem() .blockTokenSecretManager.generateToken(wrongBlock, EnumSet.of(BlockTokenSecretManager.AccessMode.READ))); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java Fri Sep 3 23:38:21 2010 @@ -163,7 +163,7 @@ public class TestBlockUnderConstruction final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len); final List<LocatedBlock> blocks = lb.getLocatedBlocks(); assertEquals(i, blocks.size()); - final Block b = blocks.get(blocks.size() - 1).getBlock(); + final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock(); assertTrue(b instanceof BlockInfoUnderConstruction); if (++i < NUM_BLOCKS) { Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java Fri Sep 3 23:38:21 2010 @@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.DFSTestUti import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas; import org.apache.commons.logging.impl.Log4JLogger; @@ -64,10 +65,10 @@ public class TestBlocksWithNotEnoughRack DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L); DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR); - Block b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); + ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); final FSNamesystem namesystem = cluster.getNamesystem(); - int numRacks = namesystem.blockManager.getNumberOfRacks(b); - NumberReplicas number = namesystem.blockManager.countNodes(b); + int numRacks = namesystem.blockManager.getNumberOfRacks(b.getLocalBlock()); + NumberReplicas number = namesystem.blockManager.countNodes(b.getLocalBlock()); int curReplicas = number.liveReplicas(); int neededReplicationSize = namesystem.blockManager.neededReplications.size(); @@ -80,8 +81,8 @@ public class TestBlocksWithNotEnoughRack (neededReplicationSize > 0) ) { LOG.info("Waiting for replication"); Thread.sleep(600); - numRacks = namesystem.blockManager.getNumberOfRacks(b); - number = namesystem.blockManager.countNodes(b); + numRacks = namesystem.blockManager.getNumberOfRacks(b.getLocalBlock()); + number = namesystem.blockManager.countNodes(b.getLocalBlock()); curReplicas = number.liveReplicas(); neededReplicationSize = namesystem.blockManager.neededReplications.size(); @@ -118,10 +119,10 @@ public class TestBlocksWithNotEnoughRack DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L); DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR); - Block b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); + ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); final FSNamesystem namesystem = cluster.getNamesystem(); - int numRacks = namesystem.blockManager.getNumberOfRacks(b); - NumberReplicas number = namesystem.blockManager.countNodes(b); + int numRacks = namesystem.blockManager.getNumberOfRacks(b.getLocalBlock()); + NumberReplicas number = namesystem.blockManager.countNodes(b.getLocalBlock()); int curReplicas = number.liveReplicas(); int neededReplicationSize = namesystem.blockManager.neededReplications.size(); @@ -136,8 +137,8 @@ public class TestBlocksWithNotEnoughRack (neededReplicationSize > 0) ) { LOG.info("Waiting for replication"); Thread.sleep(600); - numRacks = namesystem.blockManager.getNumberOfRacks(b); - number = namesystem.blockManager.countNodes(b); + numRacks = namesystem.blockManager.getNumberOfRacks(b.getLocalBlock()); + number = namesystem.blockManager.countNodes(b.getLocalBlock()); curReplicas = number.liveReplicas(); neededReplicationSize = namesystem.blockManager.neededReplications.size(); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java Fri Sep 3 23:38:21 2010 @@ -97,13 +97,13 @@ public class TestDeadDatanode { waitForDatanodeState(reg.getStorageID(), false, 20000); DatanodeProtocol dnp = cluster.getNameNode(); - Block block = new Block(0); - Block[] blocks = new Block[] { block }; + String poolId = cluster.getNamesystem().getPoolId(); + Block[] blocks = new Block[] { new Block(0) }; String[] delHints = new String[] { "" }; // Ensure blockReceived call from dead datanode is rejected with IOException try { - dnp.blockReceived(reg, blocks, delHints); + dnp.blockReceived(reg, poolId, blocks, delHints); Assert.fail("Expected IOException is not thrown"); } catch (IOException ex) { // Expected @@ -112,7 +112,7 @@ public class TestDeadDatanode { // Ensure blockReport from dead datanode is rejected with IOException long[] blockReport = new long[] { 0L, 0L, 0L }; try { - dnp.blockReport(reg, blockReport); + dnp.blockReport(reg, poolId, blockReport); Assert.fail("Expected IOException is not thrown"); } catch (IOException ex) { // Expected Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java Fri Sep 3 23:38:21 2010 @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas; import junit.framework.TestCase; @@ -51,7 +52,7 @@ public class TestNodeCount extends TestC final Path FILE_PATH = new Path("/testfile"); DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L); DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR); - Block block = DFSTestUtil.getFirstBlock(fs, FILE_PATH); + ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, FILE_PATH); // keep a copy of all datanode descriptor DatanodeDescriptor[] datanodes = @@ -80,12 +81,13 @@ public class TestNodeCount extends TestC NumberReplicas num = null; do { synchronized (namesystem) { - num = namesystem.blockManager.countNodes(block); + num = namesystem.blockManager.countNodes(block.getLocalBlock()); } } while (num.excessReplicas() == 0); // find out a non-excess node - Iterator<DatanodeDescriptor> iter = namesystem.blockManager.blocksMap.nodeIterator(block); + Iterator<DatanodeDescriptor> iter = namesystem.blockManager.blocksMap + .nodeIterator(block.getLocalBlock()); DatanodeDescriptor nonExcessDN = null; while (iter.hasNext()) { DatanodeDescriptor dn = iter.next(); @@ -107,7 +109,7 @@ public class TestNodeCount extends TestC // The block should be replicated do { - num = namesystem.blockManager.countNodes(block); + num = namesystem.blockManager.countNodes(block.getLocalBlock()); } while (num.liveReplicas() != REPLICATION_FACTOR); // restart the first datanode @@ -116,7 +118,7 @@ public class TestNodeCount extends TestC // check if excessive replica is detected do { - num = namesystem.blockManager.countNodes(block); + num = namesystem.blockManager.countNodes(block.getLocalBlock()); } while (num.excessReplicas() != 2); } finally { cluster.shutdown(); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java Fri Sep 3 23:38:21 2010 @@ -28,8 +28,8 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.TestDatanodeBlockScanner; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.DFSConfigKeys; import junit.framework.TestCase; @@ -53,8 +53,9 @@ public class TestOverReplicatedBlocks ex DFSTestUtil.waitReplication(fs, fileName, (short)3); // corrupt the block on datanode 0 - Block block = DFSTestUtil.getFirstBlock(fs, fileName); - TestDatanodeBlockScanner.corruptReplica(block.getBlockName(), 0); + ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName); + TestDatanodeBlockScanner.corruptReplica(block.getLocalBlock() + .getBlockName(), 0); DataNodeProperties dnProps = cluster.stopDataNode(0); // remove block scanner log to trigger block scanning File scanLog = new File(System.getProperty("test.build.data"), @@ -89,7 +90,8 @@ public class TestOverReplicatedBlocks ex // corrupt one won't be chosen to be excess one // without 4910 the number of live replicas would be 0: block gets lost - assertEquals(1, namesystem.blockManager.countNodes(block).liveReplicas()); + assertEquals(1, namesystem.blockManager.countNodes(block.getLocalBlock()) + .liveReplicas()); } } finally { cluster.shutdown(); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestUnderReplicatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestUnderReplicatedBlocks.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestUnderReplicatedBlocks.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestUnderReplicatedBlocks.java Fri Sep 3 23:38:21 2010 @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import junit.framework.TestCase; @@ -44,10 +44,10 @@ public class TestUnderReplicatedBlocks e // remove one replica from the blocksMap so block becomes under-replicated // but the block does not get put into the under-replicated blocks queue final FSNamesystem namesystem = cluster.getNamesystem(); - Block b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); - DatanodeDescriptor dn = namesystem.blockManager.blocksMap.nodeIterator(b).next(); - namesystem.blockManager.addToInvalidates(b, dn); - namesystem.blockManager.blocksMap.removeNode(b, dn); + ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); + DatanodeDescriptor dn = namesystem.blockManager.blocksMap.nodeIterator(b.getLocalBlock()).next(); + namesystem.blockManager.addToInvalidates(b.getLocalBlock(), dn); + namesystem.blockManager.blocksMap.removeNode(b.getLocalBlock(), dn); // increment this file's replication factor FsShell shell = new FsShell(conf); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Fri Sep 3 23:38:21 2010 @@ -27,9 +27,9 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -70,12 +70,15 @@ public class TestBlockRecovery { private DataNode dn; private Configuration conf; private final static long RECOVERY_ID = 3000L; + // TODO:FEDERATION fix pool ID + private final static String POOL_ID = "TODO"; private final static long BLOCK_ID = 1000L; private final static long GEN_STAMP = 2000L; private final static long BLOCK_LEN = 3000L; private final static long REPLICA_LEN1 = 6000L; private final static long REPLICA_LEN2 = 5000L; - private final static Block block = new Block(BLOCK_ID, BLOCK_LEN, GEN_STAMP); + private final static ExtendedBlock block = new ExtendedBlock(POOL_ID, + BLOCK_ID, BLOCK_LEN, GEN_STAMP); static { ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); @@ -447,7 +450,7 @@ public class TestBlockRecovery { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); } - dn.data.createRbw(block); + dn.data.createRbw(block.getLocalBlock()); try { dn.syncBlock(rBlock, initBlockRecords(dn)); fail("Sync should fail"); @@ -455,7 +458,7 @@ public class TestBlockRecovery { e.getMessage().startsWith("Cannot recover "); } verify(dn.namenode, never()).commitBlockSynchronization( - any(Block.class), anyLong(), anyLong(), anyBoolean(), + any(ExtendedBlock.class), anyLong(), anyLong(), anyBoolean(), anyBoolean(), any(DatanodeID[].class)); } @@ -469,7 +472,7 @@ public class TestBlockRecovery { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); } - ReplicaInPipelineInterface replicaInfo = dn.data.createRbw(block); + ReplicaInPipelineInterface replicaInfo = dn.data.createRbw(block.getLocalBlock()); BlockWriteStreams streams = null; try { streams = replicaInfo.createStreams(true, 0, 0); @@ -482,7 +485,7 @@ public class TestBlockRecovery { e.getMessage().startsWith("Cannot recover "); } verify(dn.namenode, never()).commitBlockSynchronization( - any(Block.class), anyLong(), anyLong(), anyBoolean(), + any(ExtendedBlock.class), anyLong(), anyLong(), anyBoolean(), anyBoolean(), any(DatanodeID[].class)); } finally { streams.close(); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java?rev=992508&r1=992507&r2=992508&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java Fri Sep 3 23:38:21 2010 @@ -265,7 +265,7 @@ public class TestNNLeaseRecovery { BlockInfo lastBlock = fsn.dir.getFileINode(anyString()).getLastBlock(); try { - fsn.commitBlockSynchronization(lastBlock, + fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock), recoveryId, newSize, true, false, new DatanodeID[1]); } catch (IOException ioe) { assertTrue(ioe.getMessage().startsWith("Block (=")); @@ -293,7 +293,7 @@ public class TestNNLeaseRecovery { when(lastBlock.isComplete()).thenReturn(true); try { - fsn.commitBlockSynchronization(lastBlock, + fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock), recoveryId, newSize, true, false, new DatanodeID[1]); } catch (IOException ioe) { assertTrue(ioe.getMessage().startsWith("Unexpected block (=")); @@ -321,7 +321,7 @@ public class TestNNLeaseRecovery { when(((BlockInfoUnderConstruction)lastBlock).getBlockRecoveryId()).thenReturn(recoveryId-100); try { - fsn.commitBlockSynchronization(lastBlock, + fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock), recoveryId, newSize, true, false, new DatanodeID[1]); } catch (IOException ioe) { assertTrue(ioe.getMessage().startsWith("The recovery id " + recoveryId + " does not match current recovery id " + (recoveryId-100))); @@ -349,7 +349,7 @@ public class TestNNLeaseRecovery { when(((BlockInfoUnderConstruction)lastBlock).getBlockRecoveryId()).thenReturn(recoveryId+100); try { - fsn.commitBlockSynchronization(lastBlock, + fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock), recoveryId, newSize, true, false, new DatanodeID[1]); } catch (IOException ioe) { assertTrue(ioe.getMessage().startsWith("The recovery id " + recoveryId + " does not match current recovery id " + (recoveryId+100))); @@ -378,7 +378,7 @@ public class TestNNLeaseRecovery { boolean recoveryChecked = false; try { - fsn.commitBlockSynchronization(lastBlock, + fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock), recoveryId, newSize, true, false, new DatanodeID[1]); } catch (NullPointerException ioe) { // It is fine to get NPE here because the datanodes array is empty
