Author: szetszwo
Date: Tue Oct 22 00:19:04 2013
New Revision: 1534424
URL: http://svn.apache.org/r1534424
Log:
svn merge -c -1534417 . for reverting HDFS-4885, which should be committed to
trunk.
Removed:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt?rev=1534424&r1=1534423&r2=1534424&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt
Tue Oct 22 00:19:04 2013
@@ -36,6 +36,3 @@ IMPROVEMENTS:
HDFS-4988. Datanode must support all the volumes as individual storages.
(Arpit Agarwal)
-
- HDFS-4885. Improve the verifyBlockPlacement() API in BlockPlacementPolicy.
- (Junping Du via szetszwo)
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java?rev=1534424&r1=1534423&r2=1534424&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
Tue Oct 22 00:19:04 2013
@@ -100,17 +100,18 @@ public abstract class BlockPlacementPoli
}
/**
- * Verify if the block's placement meets requirement of placement policy,
- * i.e. replicas are placed on no less than minRacks racks in the system.
+ * Verify that the block is replicated on at least minRacks different racks
+ * if there is more than minRacks rack in the system.
*
* @param srcPath the full pathname of the file to be verified
* @param lBlk block with locations
- * @param numOfReplicas replica number of file to be verified
- * @return the result of verification
- */
- abstract public BlockPlacementStatus verifyBlockPlacement(String srcPath,
- LocatedBlock lBlk,
- int numOfReplicas);
+ * @param minRacks number of racks the block should be replicated to
+ * @return the difference between the required and the actual number of racks
+ * the block is replicated to.
+ */
+ abstract public int verifyBlockPlacement(String srcPath,
+ LocatedBlock lBlk,
+ int minRacks);
/**
* Decide whether deleting the specified replica of the block still makes
* the block conform to the configured block placement policy.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1534424&r1=1534423&r2=1534424&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
Tue Oct 22 00:19:04 2013
@@ -698,22 +698,22 @@ public class BlockPlacementPolicyDefault
}
@Override
- public BlockPlacementStatus verifyBlockPlacement(String srcPath,
- LocatedBlock lBlk, int numberOfReplicas) {
+ public int verifyBlockPlacement(String srcPath,
+ LocatedBlock lBlk,
+ int minRacks) {
DatanodeInfo[] locs = lBlk.getLocations();
if (locs == null)
locs = DatanodeDescriptor.EMPTY_ARRAY;
int numRacks = clusterMap.getNumOfRacks();
if(numRacks <= 1) // only one rack
- return new BlockPlacementStatusDefault(
- Math.min(numRacks, numberOfReplicas), numRacks);
- int minRacks = Math.min(2, numberOfReplicas);
+ return 0;
+ minRacks = Math.min(minRacks, numRacks);
// 1. Check that all locations are different.
// 2. Count locations on different racks.
Set<String> racks = new TreeSet<String>();
for (DatanodeInfo dn : locs)
racks.add(dn.getNetworkLocation());
- return new BlockPlacementStatusDefault(racks.size(), minRacks);
+ return minRacks - racks.size();
}
@Override
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1534424&r1=1534423&r2=1534424&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
Tue Oct 22 00:19:04 2013
@@ -50,7 +50,6 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.net.NetUtils;
@@ -375,10 +374,9 @@ public class NamenodeFsck {
locs.length + " replica(s).");
}
// verify block placement policy
- BlockPlacementStatus blockPlacementStatus =
- BlockPlacementPolicy.getInstance(conf, null, networktopology).
- verifyBlockPlacement(path, lBlk, targetFileReplication);
- if (!blockPlacementStatus.isPlacementPolicySatisfied()) {
+ int missingRacks = BlockPlacementPolicy.getInstance(conf, null,
networktopology).
+ verifyBlockPlacement(path, lBlk,
Math.min(2,targetFileReplication));
+ if (missingRacks > 0) {
res.numMisReplicatedBlocks++;
misReplicatedPerFile++;
if (!showFiles) {
@@ -387,7 +385,9 @@ public class NamenodeFsck {
out.print(path + ": ");
}
out.println(" Replica placement policy is violated for " +
- block + ". " + blockPlacementStatus.getErrorDescription());
+ block +
+ ". Block should be additionally replicated on " +
+ missingRacks + " more rack(s).");
}
report.append(i + ". " + blkName + " len=" + block.getNumBytes());
if (locs.length == 0) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1534424&r1=1534423&r2=1534424&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Tue Oct 22 00:19:04 2013
@@ -83,6 +83,7 @@ import org.apache.log4j.RollingFileAppen
import org.junit.Test;
import com.google.common.collect.Sets;
+import org.mockito.Mockito;
import static org.mockito.Mockito.*;
/**
@@ -891,80 +892,6 @@ public class TestFsck {
}
}
}
-
- /**
- * Tests that the # of misreplaced replicas is correct
- * @throws IOException
- */
- @Test
- public void testFsckMisPlacedReplicas() throws IOException {
- // Desired replication factor
- final short REPL_FACTOR = 2;
- // Number of replicas to actually start
- short NUM_DN = 2;
- // Number of blocks to write
- final short NUM_BLOCKS = 3;
- // Set a small-ish blocksize
- final long blockSize = 512;
-
- String [] racks = {"/rack1", "/rack1"};
- String [] hosts = {"host1", "host2"};
-
- Configuration conf = new Configuration();
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
-
- MiniDFSCluster cluster = null;
- DistributedFileSystem dfs = null;
-
- try {
- // Startup a minicluster
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts)
- .racks(racks).build();
- assertNotNull("Failed Cluster Creation", cluster);
- cluster.waitClusterUp();
- dfs = (DistributedFileSystem) cluster.getFileSystem();
- assertNotNull("Failed to get FileSystem", dfs);
-
- // Create a file that will be intentionally under-replicated
- final String pathString = new String("/testfile");
- final Path path = new Path(pathString);
- long fileLen = blockSize * NUM_BLOCKS;
- DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1);
-
- // Create an under-replicated file
- NameNode namenode = cluster.getNameNode();
- NetworkTopology nettop = cluster.getNamesystem().getBlockManager()
- .getDatanodeManager().getNetworkTopology();
- // Add a new node on different rack, so previous blocks' replicas
- // are considered to be misplaced
- nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3"));
- NUM_DN++;
-
- Map<String,String[]> pmap = new HashMap<String, String[]>();
- Writer result = new StringWriter();
- PrintWriter out = new PrintWriter(result, true);
- InetAddress remoteAddress = InetAddress.getLocalHost();
- NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
- NUM_DN, (short)REPL_FACTOR, remoteAddress);
-
- // Run the fsck and check the Result
- final HdfsFileStatus file =
- namenode.getRpcServer().getFileInfo(pathString);
- assertNotNull(file);
- Result res = new Result(conf);
- fsck.check(pathString, file, res);
- // check misReplicatedBlock number.
- assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS);
- } finally {
- if(dfs != null) {
- dfs.close();
- }
- if(cluster != null) {
- cluster.shutdown();
- }
- }
- }
/** Test fsck with FileNotFound */
@Test
