Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1369798&r1=1369797&r2=1369798&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INode.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INode.java Mon Aug 6 12:03:13 2012 @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.protocol.L * This is a base INode class containing common fields for file and * directory inodes. */ -abstract class INode implements Comparable<byte[]> { +abstract class INode implements Comparable<byte[]>, FSInodeInfo { protected byte[] name; protected INodeDirectory parent; protected long modificationTime; @@ -247,6 +247,12 @@ abstract class INode implements Comparab } /** {@inheritDoc} */ + public String getFullPathName() { + // Get the full path name of this inode. + return FSDirectory.getFullPathName(this); + } + + /** {@inheritDoc} */ public String toString() { return "\"" + getLocalName() + "\":" + getPermissionStatus(); }
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1369798&r1=1369797&r2=1369798&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Mon Aug 6 12:03:13 2012 @@ -711,18 +711,18 @@ public class NameNode implements ClientP DatanodeInfo[] excludedNodes) throws IOException { - List<Node> excludedNodeList = null; + HashMap<Node, Node> excludedNodesSet = null; if (excludedNodes != null) { - // We must copy here, since this list gets modified later on - // in ReplicationTargetChooser - excludedNodeList = new ArrayList<Node>( - Arrays.<Node>asList(excludedNodes)); + excludedNodesSet = new HashMap<Node, Node>(excludedNodes.length); + for (Node node:excludedNodes) { + excludedNodesSet.put(node, node); + } } stateChangeLog.debug("*BLOCK* NameNode.addBlock: file " +src+" for "+clientName); LocatedBlock locatedBlock = namesystem.getAdditionalBlock( - src, clientName, excludedNodeList); + src, clientName, excludedNodesSet); if (locatedBlock != null) myMetrics.incrNumAddBlockOps(); return locatedBlock; Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1369798&r1=1369797&r2=1369798&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Mon Aug 6 12:03:13 2012 @@ -281,8 +281,8 @@ public class NamenodeFsck { locs.length + " replica(s)."); } // verify block placement policy - int missingRacks = ReplicationTargetChooser.verifyBlockPlacement( - lBlk, targetFileReplication, networktopology); + int missingRacks = BlockPlacementPolicy.getInstance(conf, null, networktopology). + verifyBlockPlacement(path, lBlk, Math.min(2,targetFileReplication)); if (missingRacks > 0) { res.numMisReplicatedBlocks++; misReplicatedPerFile++; Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java?rev=1369798&r1=1369797&r2=1369798&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java Mon Aug 6 12:03:13 2012 @@ -1,528 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode; - -import org.apache.commons.logging.*; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.FSConstants; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.net.NetworkTopology; -import org.apache.hadoop.net.Node; -import org.apache.hadoop.net.NodeBase; -import java.util.*; - -/** The class is responsible for choosing the desired number of targets - * for placing block replicas. - * The replica placement strategy is that if the writer is on a datanode, - * the 1st replica is placed on the local machine, - * otherwise a random datanode. The 2nd replica is placed on a datanode - * that is on a different rack. The 3rd replica is placed on a datanode - * which is on the same rack as the second replica. - */ -class ReplicationTargetChooser { - private final boolean considerLoad; - private NetworkTopology clusterMap; - private FSNamesystem fs; - - ReplicationTargetChooser(boolean considerLoad, FSNamesystem fs, - NetworkTopology clusterMap) { - this.considerLoad = considerLoad; - this.fs = fs; - this.clusterMap = clusterMap; - } - - private static class NotEnoughReplicasException extends Exception { - NotEnoughReplicasException(String msg) { - super(msg); - } - } - - /** - * choose <i>numOfReplicas</i> data nodes for <i>writer</i> to replicate - * a block with size <i>blocksize</i> - * If not, return as many as we can. - * - * @param numOfReplicas: number of replicas wanted. - * @param writer: the writer's machine, null if not in the cluster. - * @param excludedNodes: datanodesthat should not be considered targets. - * @param blocksize: size of the data to be written. - * @return array of DatanodeDescriptor instances chosen as targets - * and sorted as a pipeline. - */ - DatanodeDescriptor[] chooseTarget(int numOfReplicas, - DatanodeDescriptor writer, - List<Node> excludedNodes, - long blocksize) { - if (excludedNodes == null) { - excludedNodes = new ArrayList<Node>(); - } - - return chooseTarget(numOfReplicas, writer, - new ArrayList<DatanodeDescriptor>(), excludedNodes, blocksize); - } - - /** - * choose <i>numOfReplicas</i> data nodes for <i>writer</i> - * to re-replicate a block with size <i>blocksize</i> - * If not, return as many as we can. - * - * @param numOfReplicas: additional number of replicas wanted. - * @param writer: the writer's machine, null if not in the cluster. - * @param choosenNodes: datanodes that have been choosen as targets. - * @param excludedNodes: datanodesthat should not be considered targets. - * @param blocksize: size of the data to be written. - * @return array of DatanodeDescriptor instances chosen as target - * and sorted as a pipeline. - */ - DatanodeDescriptor[] chooseTarget(int numOfReplicas, - DatanodeDescriptor writer, - List<DatanodeDescriptor> choosenNodes, - List<Node> excludedNodes, - long blocksize) { - if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { - return new DatanodeDescriptor[0]; - } - - if (excludedNodes == null) { - excludedNodes = new ArrayList<Node>(); - } - - int clusterSize = clusterMap.getNumOfLeaves(); - int totalNumOfReplicas = choosenNodes.size()+numOfReplicas; - if (totalNumOfReplicas > clusterSize) { - numOfReplicas -= (totalNumOfReplicas-clusterSize); - totalNumOfReplicas = clusterSize; - } - - int maxNodesPerRack = - (totalNumOfReplicas-1)/clusterMap.getNumOfRacks()+2; - - List<DatanodeDescriptor> results = - new ArrayList<DatanodeDescriptor>(choosenNodes); - excludedNodes.addAll(choosenNodes); - - if (!clusterMap.contains(writer)) { - writer=null; - } - - DatanodeDescriptor localNode = chooseTarget(numOfReplicas, writer, - excludedNodes, blocksize, maxNodesPerRack, results); - - results.removeAll(choosenNodes); - - // sorting nodes to form a pipeline - return getPipeline((writer==null)?localNode:writer, - results.toArray(new DatanodeDescriptor[results.size()])); - } - - /* choose <i>numOfReplicas</i> from all data nodes */ - private DatanodeDescriptor chooseTarget(int numOfReplicas, - DatanodeDescriptor writer, - List<Node> excludedNodes, - long blocksize, - int maxNodesPerRack, - List<DatanodeDescriptor> results) { - - if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { - return writer; - } - - int numOfResults = results.size(); - boolean newBlock = (numOfResults==0); - if (writer == null && !newBlock) { - writer = (DatanodeDescriptor)results.get(0); - } - - try { - switch(numOfResults) { - case 0: - writer = chooseLocalNode(writer, excludedNodes, - blocksize, maxNodesPerRack, results); - if (--numOfReplicas == 0) { - break; - } - case 1: - chooseRemoteRack(1, results.get(0), excludedNodes, - blocksize, maxNodesPerRack, results); - if (--numOfReplicas == 0) { - break; - } - case 2: - if (clusterMap.isOnSameRack(results.get(0), results.get(1))) { - chooseRemoteRack(1, results.get(0), excludedNodes, - blocksize, maxNodesPerRack, results); - } else if (newBlock){ - chooseLocalRack(results.get(1), excludedNodes, blocksize, - maxNodesPerRack, results); - } else { - chooseLocalRack(writer, excludedNodes, blocksize, - maxNodesPerRack, results); - } - if (--numOfReplicas == 0) { - break; - } - default: - chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results); - } - } catch (NotEnoughReplicasException e) { - FSNamesystem.LOG.warn("Not able to place enough replicas, still in need of " - + numOfReplicas); - } - return writer; - } - - /* choose <i>localMachine</i> as the target. - * if <i>localMachine</i> is not availabe, - * choose a node on the same rack - * @return the choosen node - */ - private DatanodeDescriptor chooseLocalNode( - DatanodeDescriptor localMachine, - List<Node> excludedNodes, - long blocksize, - int maxNodesPerRack, - List<DatanodeDescriptor> results) - throws NotEnoughReplicasException { - // if no local machine, randomly choose one node - if (localMachine == null) - return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results); - - // otherwise try local machine first - if (!excludedNodes.contains(localMachine)) { - excludedNodes.add(localMachine); - if (isGoodTarget(localMachine, blocksize, - maxNodesPerRack, false, results)) { - results.add(localMachine); - return localMachine; - } - } - - // try a node on local rack - return chooseLocalRack(localMachine, excludedNodes, - blocksize, maxNodesPerRack, results); - } - - /* choose one node from the rack that <i>localMachine</i> is on. - * if no such node is availabe, choose one node from the rack where - * a second replica is on. - * if still no such node is available, choose a random node - * in the cluster. - * @return the choosen node - */ - private DatanodeDescriptor chooseLocalRack( - DatanodeDescriptor localMachine, - List<Node> excludedNodes, - long blocksize, - int maxNodesPerRack, - List<DatanodeDescriptor> results) - throws NotEnoughReplicasException { - // no local machine, so choose a random machine - if (localMachine == null) { - return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results); - } - - // choose one from the local rack - try { - return chooseRandom( - localMachine.getNetworkLocation(), - excludedNodes, blocksize, maxNodesPerRack, results); - } catch (NotEnoughReplicasException e1) { - // find the second replica - DatanodeDescriptor newLocal=null; - for(Iterator<DatanodeDescriptor> iter=results.iterator(); - iter.hasNext();) { - DatanodeDescriptor nextNode = iter.next(); - if (nextNode != localMachine) { - newLocal = nextNode; - break; - } - } - if (newLocal != null) { - try { - return chooseRandom( - newLocal.getNetworkLocation(), - excludedNodes, blocksize, maxNodesPerRack, results); - } catch(NotEnoughReplicasException e2) { - //otherwise randomly choose one from the network - return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results); - } - } else { - //otherwise randomly choose one from the network - return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results); - } - } - } - - /* choose <i>numOfReplicas</i> nodes from the racks - * that <i>localMachine</i> is NOT on. - * if not enough nodes are availabe, choose the remaining ones - * from the local rack - */ - - private void chooseRemoteRack(int numOfReplicas, - DatanodeDescriptor localMachine, - List<Node> excludedNodes, - long blocksize, - int maxReplicasPerRack, - List<DatanodeDescriptor> results) - throws NotEnoughReplicasException { - int oldNumOfReplicas = results.size(); - // randomly choose one node from remote racks - try { - chooseRandom(numOfReplicas, "~"+localMachine.getNetworkLocation(), - excludedNodes, blocksize, maxReplicasPerRack, results); - } catch (NotEnoughReplicasException e) { - chooseRandom(numOfReplicas-(results.size()-oldNumOfReplicas), - localMachine.getNetworkLocation(), excludedNodes, blocksize, - maxReplicasPerRack, results); - } - } - - /* Randomly choose one target from <i>nodes</i>. - * @return the choosen node - */ - private DatanodeDescriptor chooseRandom( - String nodes, - List<Node> excludedNodes, - long blocksize, - int maxNodesPerRack, - List<DatanodeDescriptor> results) - throws NotEnoughReplicasException { - DatanodeDescriptor result; - do { - DatanodeDescriptor[] selectedNodes = - chooseRandom(1, nodes, excludedNodes); - if (selectedNodes.length == 0) { - throw new NotEnoughReplicasException( - "Not able to place enough replicas"); - } - result = (DatanodeDescriptor)(selectedNodes[0]); - } while(!isGoodTarget(result, blocksize, maxNodesPerRack, results)); - results.add(result); - return result; - } - - /* Randomly choose <i>numOfReplicas</i> targets from <i>nodes</i>. - */ - private void chooseRandom(int numOfReplicas, - String nodes, - List<Node> excludedNodes, - long blocksize, - int maxNodesPerRack, - List<DatanodeDescriptor> results) - throws NotEnoughReplicasException { - boolean toContinue = true; - do { - DatanodeDescriptor[] selectedNodes = - chooseRandom(numOfReplicas, nodes, excludedNodes); - if (selectedNodes.length < numOfReplicas) { - toContinue = false; - } - for(int i=0; i<selectedNodes.length; i++) { - DatanodeDescriptor result = selectedNodes[i]; - if (isGoodTarget(result, blocksize, maxNodesPerRack, results)) { - numOfReplicas--; - results.add(result); - } - } // end of for - } while (numOfReplicas>0 && toContinue); - - if (numOfReplicas>0) { - throw new NotEnoughReplicasException( - "Not able to place enough replicas"); - } - } - - /* Randomly choose <i>numOfNodes</i> nodes from <i>scope</i>. - * @return the choosen nodes - */ - private DatanodeDescriptor[] chooseRandom(int numOfReplicas, - String nodes, - List<Node> excludedNodes) { - List<DatanodeDescriptor> results = - new ArrayList<DatanodeDescriptor>(); - int numOfAvailableNodes = - clusterMap.countNumOfAvailableNodes(nodes, excludedNodes); - numOfReplicas = (numOfAvailableNodes<numOfReplicas)? - numOfAvailableNodes:numOfReplicas; - while(numOfReplicas > 0) { - DatanodeDescriptor choosenNode = - (DatanodeDescriptor)(clusterMap.chooseRandom(nodes)); - if (!excludedNodes.contains(choosenNode)) { - results.add(choosenNode); - excludedNodes.add(choosenNode); - numOfReplicas--; - } - } - return (DatanodeDescriptor[])results.toArray( - new DatanodeDescriptor[results.size()]); - } - - /* judge if a node is a good target. - * return true if <i>node</i> has enough space, - * does not have too much load, and the rack does not have too many nodes - */ - private boolean isGoodTarget(DatanodeDescriptor node, - long blockSize, int maxTargetPerLoc, - List<DatanodeDescriptor> results) { - return isGoodTarget(node, blockSize, maxTargetPerLoc, - this.considerLoad, results); - } - - private boolean isGoodTarget(DatanodeDescriptor node, - long blockSize, int maxTargetPerLoc, - boolean considerLoad, - List<DatanodeDescriptor> results) { - Log logr = FSNamesystem.LOG; - // check if the node is (being) decommissed - if (node.isDecommissionInProgress() || node.isDecommissioned()) { - logr.debug("Node "+NodeBase.getPath(node)+ - " is not chosen because the node is (being) decommissioned"); - return false; - } - - long remaining = node.getRemaining() - - (node.getBlocksScheduled() * blockSize); - // check the remaining capacity of the target machine - if (blockSize* FSConstants.MIN_BLOCKS_FOR_WRITE>remaining) { - logr.debug("Node "+NodeBase.getPath(node)+ - " is not chosen because the node does not have enough space"); - return false; - } - - // check the communication traffic of the target machine - if (considerLoad) { - double avgLoad = 0; - int size = clusterMap.getNumOfLeaves(); - if (size != 0) { - avgLoad = (double)fs.getTotalLoad()/size; - } - if (node.getXceiverCount() > (2.0 * avgLoad)) { - logr.debug("Node "+NodeBase.getPath(node)+ - " is not chosen because the node is too busy"); - return false; - } - } - - // check if the target rack has chosen too many nodes - String rackname = node.getNetworkLocation(); - int counter=1; - for(Iterator<DatanodeDescriptor> iter = results.iterator(); - iter.hasNext();) { - Node result = iter.next(); - if (rackname.equals(result.getNetworkLocation())) { - counter++; - } - } - if (counter>maxTargetPerLoc) { - logr.debug("Node "+NodeBase.getPath(node)+ - " is not chosen because the rack has too many chosen nodes"); - return false; - } - return true; - } - - /* Return a pipeline of nodes. - * The pipeline is formed finding a shortest path that - * starts from the writer and tranverses all <i>nodes</i> - * This is basically a traveling salesman problem. - */ - private DatanodeDescriptor[] getPipeline( - DatanodeDescriptor writer, - DatanodeDescriptor[] nodes) { - if (nodes.length==0) return nodes; - - synchronized(clusterMap) { - int index=0; - if (writer == null || !clusterMap.contains(writer)) { - writer = nodes[0]; - } - for(;index<nodes.length; index++) { - DatanodeDescriptor shortestNode = nodes[index]; - int shortestDistance = clusterMap.getDistance(writer, shortestNode); - int shortestIndex = index; - for(int i=index+1; i<nodes.length; i++) { - DatanodeDescriptor currentNode = nodes[i]; - int currentDistance = clusterMap.getDistance(writer, currentNode); - if (shortestDistance>currentDistance) { - shortestDistance = currentDistance; - shortestNode = currentNode; - shortestIndex = i; - } - } - //switch position index & shortestIndex - if (index != shortestIndex) { - nodes[shortestIndex] = nodes[index]; - nodes[index] = shortestNode; - } - writer = shortestNode; - } - } - return nodes; - } - - /** - * Verify that the block is replicated on at least 2 different racks - * if there is more than one rack in the system. - * - * @param lBlk block with locations - * @param cluster - * @return 1 if the block must be relicated on additional rack, - * or 0 if the number of racks is sufficient. - */ - public static int verifyBlockPlacement(LocatedBlock lBlk, - short replication, - NetworkTopology cluster) { - int numRacks = verifyBlockPlacement(lBlk, Math.min(2,replication), cluster); - return numRacks < 0 ? 0 : numRacks; - } - - /** - * Verify that the block is replicated on at least minRacks different racks - * if there is more than minRacks rack in the system. - * - * @param lBlk block with locations - * @param minRacks number of racks the block should be replicated to - * @param cluster - * @return the difference between the required and the actual number of racks - * the block is replicated to. - */ - public static int verifyBlockPlacement(LocatedBlock lBlk, - int minRacks, - NetworkTopology cluster) { - DatanodeInfo[] locs = lBlk.getLocations(); - if (locs == null) - locs = new DatanodeInfo[0]; - int numRacks = cluster.getNumOfRacks(); - if(numRacks <= 1) // only one rack - return 0; - minRacks = Math.min(minRacks, numRacks); - // 1. Check that all locations are different. - // 2. Count locations on different racks. - Set<String> racks = new TreeSet<String>(); - for (DatanodeInfo dn : locs) - racks.add(dn.getNetworkLocation()); - return minRacks - racks.size(); - } -} //end of Replicator - Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/UnsupportedActionException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/UnsupportedActionException.java?rev=1369798&view=auto ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/UnsupportedActionException.java (added) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/UnsupportedActionException.java Mon Aug 6 12:03:13 2012 @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This exception is thrown when an operation is not supported. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class UnsupportedActionException extends IOException { + /** for java.io.Serializable */ + private static final long serialVersionUID = 1L; + + public UnsupportedActionException(String msg) { + super(msg); + } +} Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1369798&r1=1369797&r2=1369798&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Mon Aug 6 12:03:13 2012 @@ -139,7 +139,7 @@ public class NamenodeWebHdfsMethods { if (op == PutOpParam.Op.CREATE) { //choose a datanode near to client - final DatanodeInfo dn = ns.chooseDatanode(getRemoteAddress(), blocksize); + final DatanodeInfo dn = ns.chooseDatanode(path, getRemoteAddress(), blocksize); if (dn != null) { return dn; } Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java?rev=1369798&r1=1369797&r2=1369798&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java Mon Aug 6 12:03:13 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -37,7 +38,8 @@ public class TestReplicationPolicy exten private static final Configuration CONF = new Configuration(); private static final NetworkTopology cluster; private static final NameNode namenode; - private static final ReplicationTargetChooser replicator; + private static final BlockPlacementPolicy replicator; + private static final String filename = "/dummyfile.txt"; private static final DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { new DatanodeDescriptor(new DatanodeID("h1:5020"), "/d1/r1"), @@ -90,30 +92,30 @@ public class TestReplicationPolicy exten FSConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 4); // overloaded DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); - targets = replicator.chooseTarget( - 2, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget( - 3, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 3, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - targets = replicator.chooseTarget( - 4, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 4, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || @@ -134,43 +136,49 @@ public class TestReplicationPolicy exten * @throws Exception */ public void testChooseTarget2() throws Exception { - List<Node> excludedNodes; + HashMap<Node, Node> excludedNodes; DatanodeDescriptor[] targets; + BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator; + List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); - excludedNodes = new ArrayList<Node>(); - excludedNodes.add(dataNodes[1]); - targets = replicator.chooseTarget( - 0, dataNodes[0], excludedNodes, BLOCK_SIZE); + excludedNodes = new HashMap<Node, Node>(); + excludedNodes.put(dataNodes[1], dataNodes[1]); + targets = repl.chooseTarget( + 0, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 0); excludedNodes.clear(); - excludedNodes.add(dataNodes[1]); - targets = replicator.chooseTarget( - 1, dataNodes[0], excludedNodes, BLOCK_SIZE); + chosenNodes.clear(); + excludedNodes.put(dataNodes[1], dataNodes[1]); + targets = repl.chooseTarget( + 1, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); excludedNodes.clear(); - excludedNodes.add(dataNodes[1]); - targets = replicator.chooseTarget( - 2, dataNodes[0], excludedNodes, BLOCK_SIZE); + chosenNodes.clear(); + excludedNodes.put(dataNodes[1], dataNodes[1]); + targets = repl.chooseTarget( + 2, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); excludedNodes.clear(); - excludedNodes.add(dataNodes[1]); - targets = replicator.chooseTarget( - 3, dataNodes[0], excludedNodes, BLOCK_SIZE); + chosenNodes.clear(); + excludedNodes.put(dataNodes[1], dataNodes[1]); + targets = repl.chooseTarget( + 3, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); excludedNodes.clear(); - excludedNodes.add(dataNodes[1]); - targets = replicator.chooseTarget( - 4, dataNodes[0], excludedNodes, BLOCK_SIZE); + chosenNodes.clear(); + excludedNodes.put(dataNodes[1], dataNodes[1]); + targets = repl.chooseTarget( + 4, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); for(int i=1; i<4; i++) { @@ -196,30 +204,30 @@ public class TestReplicationPolicy exten (FSConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0); // no space DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[1]); - targets = replicator.chooseTarget( - 2, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[1]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget( - 3, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 3, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[1]); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget( - 4, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 4, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[1]); for(int i=1; i<4; i++) { @@ -251,23 +259,23 @@ public class TestReplicationPolicy exten } DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - targets = replicator.chooseTarget( - 2, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget( - 3, dataNodes[0], null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 3, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); @@ -291,21 +299,21 @@ public class TestReplicationPolicy exten */ public void testChooseTarget5() throws Exception { DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, NODE, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, NODE, BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, NODE, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, NODE, BLOCK_SIZE); assertEquals(targets.length, 1); - targets = replicator.chooseTarget( - 2, NODE, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, NODE, BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget( - 3, NODE, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 3, NODE, BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); @@ -323,23 +331,23 @@ public class TestReplicationPolicy exten chosenNodes.add(dataNodes[0]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget( - 2, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget( - 3, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 3, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[2])); @@ -358,17 +366,17 @@ public class TestReplicationPolicy exten chosenNodes.add(dataNodes[1]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget( - 2, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1])); @@ -387,29 +395,29 @@ public class TestReplicationPolicy exten chosenNodes.add(dataNodes[2]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget( - 0, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget( - 1, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0])); - targets = replicator.chooseTarget( - 1, dataNodes[2], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 1, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget( - 2, dataNodes[0], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget( - 2, dataNodes[2], chosenNodes, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, + 2, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); }
