Author: eli
Date: Tue May 15 19:03:41 2012
New Revision: 1338832
URL: http://svn.apache.org/viewvc?rev=1338832&view=rev
Log:
HDFS-3418. svn merge -c 1338830 from trunk
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/ (props
changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1338830
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1338830
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue May 15 19:03:41 2012
@@ -300,6 +300,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8285 HDFS changes for Use ProtoBuf for RpcPayLoadHeader (sanjay
radia)
+ HDFS-3418. Rename BlockWithLocationsProto datanodeIDs field to storageIDs.
+ (eli)
+
OPTIMIZATIONS
HDFS-2477. Optimize computing the diff between a block report and the
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1338830
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
Tue May 15 19:03:41 2012
@@ -254,11 +254,11 @@ public class PBHelper {
public static BlockWithLocationsProto convert(BlockWithLocations blk) {
return BlockWithLocationsProto.newBuilder()
.setBlock(convert(blk.getBlock()))
- .addAllDatanodeIDs(Arrays.asList(blk.getDatanodes())).build();
+ .addAllStorageIDs(Arrays.asList(blk.getStorageIDs())).build();
}
public static BlockWithLocations convert(BlockWithLocationsProto b) {
- return new BlockWithLocations(convert(b.getBlock()), b.getDatanodeIDsList()
+ return new BlockWithLocations(convert(b.getBlock()), b.getStorageIDsList()
.toArray(new String[0]));
}
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
Tue May 15 19:03:41 2012
@@ -205,6 +205,7 @@ public class Balancer {
private Map<Block, BalancerBlock> globalBlockList
= new HashMap<Block, BalancerBlock>();
private MovedBlocks movedBlocks = new MovedBlocks();
+ // Map storage IDs to BalancerDatanodes
private Map<String, BalancerDatanode> datanodes
= new HashMap<String, BalancerDatanode>();
@@ -620,8 +621,8 @@ public class Balancer {
synchronized (block) {
// update locations
- for ( String location : blk.getDatanodes() ) {
- BalancerDatanode datanode = datanodes.get(location);
+ for ( String storageID : blk.getStorageIDs() ) {
+ BalancerDatanode datanode = datanodes.get(storageID);
if (datanode != null) { // not an unknown datanode
block.addLocation(datanode);
}
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
Tue May 15 19:03:41 2012
@@ -21,9 +21,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.Block;
-/** A class to implement an array of BlockLocations
- * It provide efficient customized serialization/deserialization methods
- * in stead of using the default array (de)serialization provided by RPC
+/**
+ * Maintains an array of blocks and their corresponding storage IDs.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -36,12 +35,12 @@ public class BlocksWithLocations {
@InterfaceStability.Evolving
public static class BlockWithLocations {
Block block;
- String datanodeIDs[];
+ String storageIDs[];
/** constructor */
- public BlockWithLocations(Block b, String[] datanodes) {
- block = b;
- datanodeIDs = datanodes;
+ public BlockWithLocations(Block block, String[] storageIDs) {
+ this.block = block;
+ this.storageIDs = storageIDs;
}
/** get the block */
@@ -50,15 +49,15 @@ public class BlocksWithLocations {
}
/** get the block's locations */
- public String[] getDatanodes() {
- return datanodeIDs;
+ public String[] getStorageIDs() {
+ return storageIDs;
}
}
private BlockWithLocations[] blocks;
/** Constructor with one parameter */
- public BlocksWithLocations( BlockWithLocations[] blocks ) {
+ public BlocksWithLocations(BlockWithLocations[] blocks) {
this.blocks = blocks;
}
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1338830
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
Tue May 15 19:03:41 2012
@@ -274,7 +274,7 @@ message BlockProto {
*/
message BlockWithLocationsProto {
required BlockProto block = 1; // Block
- repeated string datanodeIDs = 2; // Datanodes with replicas of the block
+ repeated string storageIDs = 2; // Datanodes with replicas of the block
}
/**
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1338830
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1338830
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1338830
Propchange:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1338830
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
Tue May 15 19:03:41 2012
@@ -101,18 +101,18 @@ public class TestGetBlocks extends TestC
BlockWithLocations[] locs;
locs = namenode.getBlocks(dataNodes[0], fileLen).getBlocks();
assertEquals(locs.length, 2);
- assertEquals(locs[0].getDatanodes().length, 2);
- assertEquals(locs[1].getDatanodes().length, 2);
+ assertEquals(locs[0].getStorageIDs().length, 2);
+ assertEquals(locs[1].getStorageIDs().length, 2);
// get blocks of size BlockSize from dataNodes[0]
locs = namenode.getBlocks(dataNodes[0], DEFAULT_BLOCK_SIZE).getBlocks();
assertEquals(locs.length, 1);
- assertEquals(locs[0].getDatanodes().length, 2);
+ assertEquals(locs[0].getStorageIDs().length, 2);
// get blocks of size 1 from dataNodes[0]
locs = namenode.getBlocks(dataNodes[0], 1).getBlocks();
assertEquals(locs.length, 1);
- assertEquals(locs[0].getDatanodes().length, 2);
+ assertEquals(locs[0].getStorageIDs().length, 2);
// get blocks of size 0 from dataNodes[0]
getBlocksWithException(namenode, dataNodes[0], 0);
Modified:
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java?rev=1338832&r1=1338831&r2=1338832&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
(original)
+++
hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
Tue May 15 19:03:41 2012
@@ -161,7 +161,7 @@ public class TestPBHelper {
private void compare(BlockWithLocations locs1, BlockWithLocations locs2) {
assertEquals(locs1.getBlock(), locs2.getBlock());
- assertTrue(Arrays.equals(locs1.getDatanodes(), locs2.getDatanodes()));
+ assertTrue(Arrays.equals(locs1.getStorageIDs(), locs2.getStorageIDs()));
}
@Test
