HDFS-7743. Code cleanup of BlockInfo and rename BlockInfo to
BlockInfoContiguous. Contributed by Jing Zhao.
(cherry picked from commit 1382ae525c67bf95d8f3a436b547dbc72cfbb177)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e49b40f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e49b40f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e49b40f
Branch: refs/heads/branch-2
Commit: 1e49b40f1f22a70f26928a8a46a32a4d0432af0b
Parents: 6933975
Author: Jing Zhao <[email protected]>
Authored: Sun Feb 8 11:51:44 2015 -0800
Committer: Jing Zhao <[email protected]>
Committed: Sun Feb 8 12:16:09 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockCollection.java | 10 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 406 ------------------
.../blockmanagement/BlockInfoContiguous.java | 415 +++++++++++++++++++
.../BlockInfoContiguousUnderConstruction.java | 400 ++++++++++++++++++
.../BlockInfoUnderConstruction.java | 402 ------------------
.../server/blockmanagement/BlockManager.java | 160 +++----
.../hdfs/server/blockmanagement/BlocksMap.java | 32 +-
.../CacheReplicationMonitor.java | 14 +-
.../blockmanagement/DatanodeDescriptor.java | 28 +-
.../server/blockmanagement/DatanodeManager.java | 4 +-
.../blockmanagement/DatanodeStorageInfo.java | 22 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 6 +-
.../hdfs/server/namenode/FSDirectory.java | 16 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 8 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 26 +-
.../hdfs/server/namenode/FSImageFormat.java | 18 +-
.../server/namenode/FSImageFormatPBINode.java | 14 +-
.../server/namenode/FSImageSerialization.java | 10 +-
.../hdfs/server/namenode/FSNamesystem.java | 74 ++--
.../namenode/FileUnderConstructionFeature.java | 17 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 84 ++--
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 4 +-
.../snapshot/FSImageFormatPBSnapshot.java | 8 +-
.../hdfs/server/namenode/snapshot/FileDiff.java | 10 +-
.../server/namenode/snapshot/FileDiffList.java | 18 +-
.../snapshot/FileWithSnapshotFeature.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 12 +-
.../server/blockmanagement/TestBlockInfo.java | 14 +-
.../TestBlockInfoUnderConstruction.java | 4 +-
.../blockmanagement/TestBlockManager.java | 20 +-
.../blockmanagement/TestDatanodeDescriptor.java | 4 +-
.../blockmanagement/TestHeartbeatHandling.java | 6 +-
.../hdfs/server/namenode/CreateEditsLog.java | 8 +-
.../hdfs/server/namenode/TestAddBlock.java | 12 +-
.../namenode/TestBlockUnderConstruction.java | 10 +-
.../TestCommitBlockSynchronization.java | 12 +-
.../hdfs/server/namenode/TestEditLog.java | 4 +-
.../hdfs/server/namenode/TestFSImage.java | 4 +-
.../hdfs/server/namenode/TestFileTruncate.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../hdfs/server/namenode/TestINodeFile.java | 4 +-
.../namenode/ha/TestPipelinesFailover.java | 6 -
.../namenode/ha/TestRetryCacheWithHA.java | 6 +-
.../namenode/snapshot/SnapshotTestHelper.java | 10 +-
.../snapshot/TestSnapshotBlocksMap.java | 24 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 16 +-
49 files changed, 1213 insertions(+), 1196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e49b40f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 56f3222..86a43e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -307,6 +307,9 @@ Release 2.7.0 - UNRELEASED
tests such as truncate with HA setup, negative tests, truncate with other
operations and multiple truncates. (szetszwo)
+ HDFS-7743. Code cleanup of BlockInfo and rename BlockInfo to
+ BlockInfoContiguous. (jing9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e49b40f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index d095a1b..1547611 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -31,7 +31,7 @@ public interface BlockCollection {
/**
* Get the last block of the collection.
*/
- public BlockInfo getLastBlock();
+ public BlockInfoContiguous getLastBlock();
/**
* Get content summary.
@@ -46,7 +46,7 @@ public interface BlockCollection {
/**
* Get the blocks.
*/
- public BlockInfo[] getBlocks();
+ public BlockInfoContiguous[] getBlocks();
/**
* Get preferred block size for the collection
@@ -73,14 +73,14 @@ public interface BlockCollection {
/**
* Set the block at the given index.
*/
- public void setBlock(int index, BlockInfo blk);
+ public void setBlock(int index, BlockInfoContiguous blk);
/**
* Convert the last block of the collection to an under-construction block
* and set the locations.
*/
- public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock,
- DatanodeStorageInfo[] locations) throws IOException;
+ public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfoContiguous
lastBlock,
+ DatanodeStorageInfo[] targets) throws IOException;
/**
* @return whether the block collection is under construction.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e49b40f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
deleted file mode 100644
index 18f45e7..0000000
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.util.LinkedList;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.util.LightWeightGSet;
-
-/**
- * BlockInfo class maintains for a given block
- * the {@link INodeFile} it is part of and datanodes where the replicas of
- * the block are stored.
- * BlockInfo class maintains for a given block
- * the {@link BlockCollection} it is part of and datanodes where the replicas
of
- * the block are stored.
- */
[email protected]
-public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
- public static final BlockInfo[] EMPTY_ARRAY = {};
-
- private BlockCollection bc;
-
- /** For implementing {@link LightWeightGSet.LinkedElement} interface */
- private LightWeightGSet.LinkedElement nextLinkedElement;
-
- /**
- * This array contains triplets of references. For each i-th storage, the
- * block belongs to triplets[3*i] is the reference to the
- * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
- * references to the previous and the next blocks, respectively, in the list
- * of blocks belonging to this storage.
- *
- * Using previous and next in Object triplets is done instead of a
- * {@link LinkedList} list to efficiently use memory. With LinkedList the
cost
- * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
- * bytes using the triplets.
- */
- private Object[] triplets;
-
- /**
- * Construct an entry for blocksmap
- * @param replication the block's replication factor
- */
- public BlockInfo(short replication) {
- this.triplets = new Object[3*replication];
- this.bc = null;
- }
-
- public BlockInfo(Block blk, short replication) {
- super(blk);
- this.triplets = new Object[3*replication];
- this.bc = null;
- }
-
- /**
- * Copy construction.
- * This is used to convert BlockInfoUnderConstruction
- * @param from BlockInfo to copy from.
- */
- protected BlockInfo(BlockInfo from) {
- this(from, from.bc.getBlockReplication());
- this.bc = from.bc;
- }
-
- public BlockCollection getBlockCollection() {
- return bc;
- }
-
- public void setBlockCollection(BlockCollection bc) {
- this.bc = bc;
- }
-
- public DatanodeDescriptor getDatanode(int index) {
- DatanodeStorageInfo storage = getStorageInfo(index);
- return storage == null ? null : storage.getDatanodeDescriptor();
- }
-
- DatanodeStorageInfo getStorageInfo(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- return (DatanodeStorageInfo)triplets[index*3];
- }
-
- private BlockInfo getPrevious(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
- BlockInfo info = (BlockInfo)triplets[index*3+1];
- assert info == null ||
- info.getClass().getName().startsWith(BlockInfo.class.getName()) :
- "BlockInfo is expected at " + index*3;
- return info;
- }
-
- BlockInfo getNext(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
- BlockInfo info = (BlockInfo)triplets[index*3+2];
- assert info == null ||
- info.getClass().getName().startsWith(BlockInfo.class.getName()) :
- "BlockInfo is expected at " + index*3;
- return info;
- }
-
- private void setStorageInfo(int index, DatanodeStorageInfo storage) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- triplets[index*3] = storage;
- }
-
- /**
- * Return the previous block on the block list for the datanode at
- * position index. Set the previous block on the list to "to".
- *
- * @param index - the datanode index
- * @param to - block to be set to previous on the list of blocks
- * @return current previous block on the list of blocks
- */
- private BlockInfo setPrevious(int index, BlockInfo to) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+1 < triplets.length : "Index is out of
bound";
- BlockInfo info = (BlockInfo)triplets[index*3+1];
- triplets[index*3+1] = to;
- return info;
- }
-
- /**
- * Return the next block on the block list for the datanode at
- * position index. Set the next block on the list to "to".
- *
- * @param index - the datanode index
- * @param to - block to be set to next on the list of blocks
- * * @return current next block on the list of blocks
- */
- private BlockInfo setNext(int index, BlockInfo to) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+2 < triplets.length : "Index is out of
bound";
- BlockInfo info = (BlockInfo)triplets[index*3+2];
- triplets[index*3+2] = to;
- return info;
- }
-
- public int getCapacity() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- return triplets.length / 3;
- }
-
- /**
- * Ensure that there is enough space to include num more triplets.
- * @return first free triplet index.
- */
- private int ensureCapacity(int num) {
- assert this.triplets != null : "BlockInfo is not initialized";
- int last = numNodes();
- if(triplets.length >= (last+num)*3)
- return last;
- /* Not enough space left. Create a new array. Should normally
- * happen only when replication is manually increased by the user. */
- Object[] old = triplets;
- triplets = new Object[(last+num)*3];
- System.arraycopy(old, 0, triplets, 0, last*3);
- return last;
- }
-
- /**
- * Count the number of data-nodes the block belongs to.
- */
- public int numNodes() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- for(int idx = getCapacity()-1; idx >= 0; idx--) {
- if(getDatanode(idx) != null)
- return idx+1;
- }
- return 0;
- }
-
- /**
- * Add a {@link DatanodeStorageInfo} location for a block
- */
- boolean addStorage(DatanodeStorageInfo storage) {
- // find the last null node
- int lastNode = ensureCapacity(1);
- setStorageInfo(lastNode, storage);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
-
- /**
- * Remove {@link DatanodeStorageInfo} location for a block
- */
- boolean removeStorage(DatanodeStorageInfo storage) {
- int dnIndex = findStorageInfo(storage);
- if(dnIndex < 0) // the node is not found
- return false;
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is still in the list and must be removed first.";
- // find the last not null node
- int lastNode = numNodes()-1;
- // replace current node triplet by the lastNode one
- setStorageInfo(dnIndex, getStorageInfo(lastNode));
- setNext(dnIndex, getNext(lastNode));
- setPrevious(dnIndex, getPrevious(lastNode));
- // set the last triplet to null
- setStorageInfo(lastNode, null);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
-
- /**
- * Find specified DatanodeDescriptor.
- * @return index or -1 if not found.
- */
- boolean findDatanode(DatanodeDescriptor dn) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeDescriptor cur = getDatanode(idx);
- if(cur == dn) {
- return true;
- }
- if(cur == null) {
- break;
- }
- }
- return false;
- }
- /**
- * Find specified DatanodeStorageInfo.
- * @return DatanodeStorageInfo or null if not found.
- */
- DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeStorageInfo cur = getStorageInfo(idx);
- if(cur == null)
- break;
- if(cur.getDatanodeDescriptor() == dn)
- return cur;
- }
- return null;
- }
-
- /**
- * Find specified DatanodeStorageInfo.
- * @return index or -1 if not found.
- */
- int findStorageInfo(DatanodeStorageInfo storageInfo) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeStorageInfo cur = getStorageInfo(idx);
- if(cur == storageInfo)
- return idx;
- if(cur == null)
- break;
- }
- return -1;
- }
-
- /**
- * Insert this block into the head of the list of blocks
- * related to the specified DatanodeStorageInfo.
- * If the head is null then form a new list.
- * @return current block as the new head of the list.
- */
- BlockInfo listInsert(BlockInfo head, DatanodeStorageInfo storage) {
- int dnIndex = this.findStorageInfo(storage);
- assert dnIndex >= 0 : "Data node is not found: current";
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is already in the list and cannot be inserted.";
- this.setPrevious(dnIndex, null);
- this.setNext(dnIndex, head);
- if(head != null)
- head.setPrevious(head.findStorageInfo(storage), this);
- return this;
- }
-
- /**
- * Remove this block from the list of blocks
- * related to the specified DatanodeStorageInfo.
- * If this block is the head of the list then return the next block as
- * the new head.
- * @return the new head of the list or null if the list becomes
- * empy after deletion.
- */
- BlockInfo listRemove(BlockInfo head, DatanodeStorageInfo storage) {
- if(head == null)
- return null;
- int dnIndex = this.findStorageInfo(storage);
- if(dnIndex < 0) // this block is not on the data-node list
- return head;
-
- BlockInfo next = this.getNext(dnIndex);
- BlockInfo prev = this.getPrevious(dnIndex);
- this.setNext(dnIndex, null);
- this.setPrevious(dnIndex, null);
- if(prev != null)
- prev.setNext(prev.findStorageInfo(storage), next);
- if(next != null)
- next.setPrevious(next.findStorageInfo(storage), prev);
- if(this == head) // removing the head
- head = next;
- return head;
- }
-
- /**
- * Remove this block from the list of blocks related to the specified
- * DatanodeDescriptor. Insert it into the head of the list of blocks.
- *
- * @return the new head of the list.
- */
- public BlockInfo moveBlockToHead(BlockInfo head, DatanodeStorageInfo storage,
- int curIndex, int headIndex) {
- if (head == this) {
- return this;
- }
- BlockInfo next = this.setNext(curIndex, head);
- BlockInfo prev = this.setPrevious(curIndex, null);
-
- head.setPrevious(headIndex, this);
- prev.setNext(prev.findStorageInfo(storage), next);
- if (next != null)
- next.setPrevious(next.findStorageInfo(storage), prev);
- return this;
- }
-
- /**
- * BlockInfo represents a block that is not being constructed.
- * In order to start modifying the block, the BlockInfo should be converted
- * to {@link BlockInfoUnderConstruction}.
- * @return {@link BlockUCState#COMPLETE}
- */
- public BlockUCState getBlockUCState() {
- return BlockUCState.COMPLETE;
- }
-
- /**
- * Is this block complete?
- *
- * @return true if the state of the block is {@link BlockUCState#COMPLETE}
- */
- public boolean isComplete() {
- return getBlockUCState().equals(BlockUCState.COMPLETE);
- }
-
- /**
- * Convert a complete block to an under construction block.
- * @return BlockInfoUnderConstruction - an under construction block.
- */
- public BlockInfoUnderConstruction convertToBlockUnderConstruction(
- BlockUCState s, DatanodeStorageInfo[] targets) {
- if(isComplete()) {
- BlockInfoUnderConstruction ucBlock = new BlockInfoUnderConstruction(this,
- getBlockCollection().getBlockReplication(), s, targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
- // the block is already under construction
- BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)this;
- ucBlock.setBlockUCState(s);
- ucBlock.setExpectedLocations(targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
-
- @Override
- public int hashCode() {
- // Super implementation is sufficient
- return super.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
- @Override
- public LightWeightGSet.LinkedElement getNext() {
- return nextLinkedElement;
- }
-
- @Override
- public void setNext(LightWeightGSet.LinkedElement next) {
- this.nextLinkedElement = next;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e49b40f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
new file mode 100644
index 0000000..1d3df26
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -0,0 +1,415 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.util.LightWeightGSet;
+
+/**
+ * BlockInfo class maintains for a given block
+ * the {@link INodeFile} it is part of and datanodes where the replicas of
+ * the block are stored.
+ * BlockInfo class maintains for a given block
+ * the {@link BlockCollection} it is part of and datanodes where the replicas
of
+ * the block are stored.
+ */
[email protected]
+public class BlockInfoContiguous extends Block
+ implements LightWeightGSet.LinkedElement {
+ public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
+
+ private BlockCollection bc;
+
+ /** For implementing {@link LightWeightGSet.LinkedElement} interface */
+ private LightWeightGSet.LinkedElement nextLinkedElement;
+
+ /**
+ * This array contains triplets of references. For each i-th storage, the
+ * block belongs to triplets[3*i] is the reference to the
+ * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
+ * references to the previous and the next blocks, respectively, in the list
+ * of blocks belonging to this storage.
+ *
+ * Using previous and next in Object triplets is done instead of a
+ * {@link LinkedList} list to efficiently use memory. With LinkedList the
cost
+ * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
+ * bytes using the triplets.
+ */
+ private Object[] triplets;
+
+ /**
+ * Construct an entry for blocksmap
+ * @param replication the block's replication factor
+ */
+ public BlockInfoContiguous(short replication) {
+ this.triplets = new Object[3*replication];
+ this.bc = null;
+ }
+
+ public BlockInfoContiguous(Block blk, short replication) {
+ super(blk);
+ this.triplets = new Object[3*replication];
+ this.bc = null;
+ }
+
+ /**
+ * Copy construction.
+ * This is used to convert BlockInfoUnderConstruction
+ * @param from BlockInfo to copy from.
+ */
+ protected BlockInfoContiguous(BlockInfoContiguous from) {
+ this(from, from.bc.getBlockReplication());
+ this.bc = from.bc;
+ }
+
+ public BlockCollection getBlockCollection() {
+ return bc;
+ }
+
+ public void setBlockCollection(BlockCollection bc) {
+ this.bc = bc;
+ }
+
+ public DatanodeDescriptor getDatanode(int index) {
+ DatanodeStorageInfo storage = getStorageInfo(index);
+ return storage == null ? null : storage.getDatanodeDescriptor();
+ }
+
+ DatanodeStorageInfo getStorageInfo(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+ return (DatanodeStorageInfo)triplets[index*3];
+ }
+
+ private BlockInfoContiguous getPrevious(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+ BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
+ assert info == null ||
+
info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) :
+ "BlockInfo is expected at " + index*3;
+ return info;
+ }
+
+ BlockInfoContiguous getNext(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+ BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
+ assert info == null || info.getClass().getName().startsWith(
+ BlockInfoContiguous.class.getName()) :
+ "BlockInfo is expected at " + index*3;
+ return info;
+ }
+
+ private void setStorageInfo(int index, DatanodeStorageInfo storage) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+ triplets[index*3] = storage;
+ }
+
+ /**
+ * Return the previous block on the block list for the datanode at
+ * position index. Set the previous block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to previous on the list of blocks
+ * @return current previous block on the list of blocks
+ */
+ private BlockInfoContiguous setPrevious(int index, BlockInfoContiguous to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+ BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
+ triplets[index*3+1] = to;
+ return info;
+ }
+
+ /**
+ * Return the next block on the block list for the datanode at
+ * position index. Set the next block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to next on the list of blocks
+ * * @return current next block on the list of blocks
+ */
+ private BlockInfoContiguous setNext(int index, BlockInfoContiguous to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+ BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
+ triplets[index*3+2] = to;
+ return info;
+ }
+
+ public int getCapacity() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ return triplets.length / 3;
+ }
+
+ /**
+ * Ensure that there is enough space to include num more triplets.
+ * @return first free triplet index.
+ */
+ private int ensureCapacity(int num) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ int last = numNodes();
+ if(triplets.length >= (last+num)*3)
+ return last;
+ /* Not enough space left. Create a new array. Should normally
+ * happen only when replication is manually increased by the user. */
+ Object[] old = triplets;
+ triplets = new Object[(last+num)*3];
+ System.arraycopy(old, 0, triplets, 0, last*3);
+ return last;
+ }
+
+ /**
+ * Count the number of data-nodes the block belongs to.
+ */
+ public int numNodes() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ for(int idx = getCapacity()-1; idx >= 0; idx--) {
+ if(getDatanode(idx) != null)
+ return idx+1;
+ }
+ return 0;
+ }
+
+ /**
+ * Add a {@link DatanodeStorageInfo} location for a block
+ */
+ boolean addStorage(DatanodeStorageInfo storage) {
+ // find the last null node
+ int lastNode = ensureCapacity(1);
+ setStorageInfo(lastNode, storage);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ /**
+ * Remove {@link DatanodeStorageInfo} location for a block
+ */
+ boolean removeStorage(DatanodeStorageInfo storage) {
+ int dnIndex = findStorageInfo(storage);
+ if(dnIndex < 0) // the node is not found
+ return false;
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is still in the list and must be removed first.";
+ // find the last not null node
+ int lastNode = numNodes()-1;
+ // replace current node triplet by the lastNode one
+ setStorageInfo(dnIndex, getStorageInfo(lastNode));
+ setNext(dnIndex, getNext(lastNode));
+ setPrevious(dnIndex, getPrevious(lastNode));
+ // set the last triplet to null
+ setStorageInfo(lastNode, null);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ /**
+ * Find specified DatanodeDescriptor.
+ * @return index or -1 if not found.
+ */
+ boolean findDatanode(DatanodeDescriptor dn) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeDescriptor cur = getDatanode(idx);
+ if(cur == dn) {
+ return true;
+ }
+ if(cur == null) {
+ break;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Find specified DatanodeStorageInfo.
+ * @return DatanodeStorageInfo or null if not found.
+ */
+ DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if(cur == null)
+ break;
+ if(cur.getDatanodeDescriptor() == dn)
+ return cur;
+ }
+ return null;
+ }
+
+ /**
+ * Find specified DatanodeStorageInfo.
+ * @return index or -1 if not found.
+ */
+ int findStorageInfo(DatanodeStorageInfo storageInfo) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if (cur == storageInfo) {
+ return idx;
+ }
+ if (cur == null) {
+ break;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Insert this block into the head of the list of blocks
+ * related to the specified DatanodeStorageInfo.
+ * If the head is null then form a new list.
+ * @return current block as the new head of the list.
+ */
+ BlockInfoContiguous listInsert(BlockInfoContiguous head,
+ DatanodeStorageInfo storage) {
+ int dnIndex = this.findStorageInfo(storage);
+ assert dnIndex >= 0 : "Data node is not found: current";
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is already in the list and cannot be inserted.";
+ this.setPrevious(dnIndex, null);
+ this.setNext(dnIndex, head);
+ if(head != null)
+ head.setPrevious(head.findStorageInfo(storage), this);
+ return this;
+ }
+
+ /**
+ * Remove this block from the list of blocks
+ * related to the specified DatanodeStorageInfo.
+ * If this block is the head of the list then return the next block as
+ * the new head.
+ * @return the new head of the list or null if the list becomes
+ * empy after deletion.
+ */
+ BlockInfoContiguous listRemove(BlockInfoContiguous head,
+ DatanodeStorageInfo storage) {
+ if(head == null)
+ return null;
+ int dnIndex = this.findStorageInfo(storage);
+ if(dnIndex < 0) // this block is not on the data-node list
+ return head;
+
+ BlockInfoContiguous next = this.getNext(dnIndex);
+ BlockInfoContiguous prev = this.getPrevious(dnIndex);
+ this.setNext(dnIndex, null);
+ this.setPrevious(dnIndex, null);
+ if(prev != null)
+ prev.setNext(prev.findStorageInfo(storage), next);
+ if(next != null)
+ next.setPrevious(next.findStorageInfo(storage), prev);
+ if(this == head) // removing the head
+ head = next;
+ return head;
+ }
+
+ /**
+ * Remove this block from the list of blocks related to the specified
+ * DatanodeDescriptor. Insert it into the head of the list of blocks.
+ *
+ * @return the new head of the list.
+ */
+ public BlockInfoContiguous moveBlockToHead(BlockInfoContiguous head,
+ DatanodeStorageInfo storage, int curIndex, int headIndex) {
+ if (head == this) {
+ return this;
+ }
+ BlockInfoContiguous next = this.setNext(curIndex, head);
+ BlockInfoContiguous prev = this.setPrevious(curIndex, null);
+
+ head.setPrevious(headIndex, this);
+ prev.setNext(prev.findStorageInfo(storage), next);
+ if (next != null) {
+ next.setPrevious(next.findStorageInfo(storage), prev);
+ }
+ return this;
+ }
+
+ /**
+ * BlockInfo represents a block that is not being constructed.
+ * In order to start modifying the block, the BlockInfo should be converted
+ * to {@link BlockInfoContiguousUnderConstruction}.
+ * @return {@link BlockUCState#COMPLETE}
+ */
+ public BlockUCState getBlockUCState() {
+ return BlockUCState.COMPLETE;
+ }
+
+ /**
+ * Is this block complete?
+ *
+ * @return true if the state of the block is {@link BlockUCState#COMPLETE}
+ */
+ public boolean isComplete() {
+ return getBlockUCState().equals(BlockUCState.COMPLETE);
+ }
+
+ /**
+ * Convert a complete block to an under construction block.
+ * @return BlockInfoUnderConstruction - an under construction block.
+ */
+ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
+ BlockUCState s, DatanodeStorageInfo[] targets) {
+ if(isComplete()) {
+ BlockInfoContiguousUnderConstruction ucBlock =
+ new BlockInfoContiguousUnderConstruction(this,
+ getBlockCollection().getBlockReplication(), s, targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+ // the block is already under construction
+ BlockInfoContiguousUnderConstruction ucBlock =
+ (BlockInfoContiguousUnderConstruction)this;
+ ucBlock.setBlockUCState(s);
+ ucBlock.setExpectedLocations(targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+
+ @Override
+ public int hashCode() {
+ // Super implementation is sufficient
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // Sufficient to rely on super's implementation
+ return (this == obj) || super.equals(obj);
+ }
+
+ @Override
+ public LightWeightGSet.LinkedElement getNext() {
+ return nextLinkedElement;
+ }
+
+ @Override
+ public void setNext(LightWeightGSet.LinkedElement next) {
+ this.nextLinkedElement = next;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e49b40f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
new file mode 100644
index 0000000..91b76cc
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -0,0 +1,400 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+
+/**
+ * Represents a block that is currently being constructed.<br>
+ * This is usually the last block of a file opened for write or append.
+ */
+public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
+ /** Block state. See {@link BlockUCState} */
+ private BlockUCState blockUCState;
+
+ /**
+ * Block replicas as assigned when the block was allocated.
+ * This defines the pipeline order.
+ */
+ private List<ReplicaUnderConstruction> replicas;
+
+ /**
+ * Index of the primary data node doing the recovery. Useful for log
+ * messages.
+ */
+ private int primaryNodeIndex = -1;
+
+ /**
+ * The new generation stamp, which this block will have
+ * after the recovery succeeds. Also used as a recovery id to identify
+ * the right recovery if any of the abandoned recoveries re-appear.
+ */
+ private long blockRecoveryId = 0;
+
+ /**
+ * The block source to use in the event of copy-on-write truncate.
+ */
+ private Block truncateBlock;
+
+ /**
+ * ReplicaUnderConstruction contains information about replicas while
+ * they are under construction.
+ * The GS, the length and the state of the replica is as reported by
+ * the data-node.
+ * It is not guaranteed, but expected, that data-nodes actually have
+ * corresponding replicas.
+ */
+ static class ReplicaUnderConstruction extends Block {
+ private final DatanodeStorageInfo expectedLocation;
+ private ReplicaState state;
+ private boolean chosenAsPrimary;
+
+ ReplicaUnderConstruction(Block block,
+ DatanodeStorageInfo target,
+ ReplicaState state) {
+ super(block);
+ this.expectedLocation = target;
+ this.state = state;
+ this.chosenAsPrimary = false;
+ }
+
+ /**
+ * Expected block replica location as assigned when the block was
allocated.
+ * This defines the pipeline order.
+ * It is not guaranteed, but expected, that the data-node actually has
+ * the replica.
+ */
+ private DatanodeStorageInfo getExpectedStorageLocation() {
+ return expectedLocation;
+ }
+
+ /**
+ * Get replica state as reported by the data-node.
+ */
+ ReplicaState getState() {
+ return state;
+ }
+
+ /**
+ * Whether the replica was chosen for recovery.
+ */
+ boolean getChosenAsPrimary() {
+ return chosenAsPrimary;
+ }
+
+ /**
+ * Set replica state.
+ */
+ void setState(ReplicaState s) {
+ state = s;
+ }
+
+ /**
+ * Set whether this replica was chosen for recovery.
+ */
+ void setChosenAsPrimary(boolean chosenAsPrimary) {
+ this.chosenAsPrimary = chosenAsPrimary;
+ }
+
+ /**
+ * Is data-node the replica belongs to alive.
+ */
+ boolean isAlive() {
+ return expectedLocation.getDatanodeDescriptor().isAlive;
+ }
+
+ @Override // Block
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override // Block
+ public boolean equals(Object obj) {
+ // Sufficient to rely on super's implementation
+ return (this == obj) || super.equals(obj);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder b = new StringBuilder(50);
+ appendStringTo(b);
+ return b.toString();
+ }
+
+ @Override
+ public void appendStringTo(StringBuilder sb) {
+ sb.append("ReplicaUC[")
+ .append(expectedLocation)
+ .append("|")
+ .append(state)
+ .append("]");
+ }
+ }
+
+ /**
+ * Create block and set its state to
+ * {@link BlockUCState#UNDER_CONSTRUCTION}.
+ */
+ public BlockInfoContiguousUnderConstruction(Block blk, short replication) {
+ this(blk, replication, BlockUCState.UNDER_CONSTRUCTION, null);
+ }
+
+ /**
+ * Create a block that is currently being constructed.
+ */
+ public BlockInfoContiguousUnderConstruction(Block blk, short replication,
BlockUCState state, DatanodeStorageInfo[] targets) {
+ super(blk, replication);
+ assert getBlockUCState() != BlockUCState.COMPLETE :
+ "BlockInfoUnderConstruction cannot be in COMPLETE state";
+ this.blockUCState = state;
+ setExpectedLocations(targets);
+ }
+
+ /**
+ * Convert an under construction block to a complete block.
+ *
+ * @return BlockInfo - a complete block.
+ * @throws IOException if the state of the block
+ * (the generation stamp and the length) has not been committed by
+ * the client or it does not have at least a minimal number of replicas
+ * reported from data-nodes.
+ */
+ BlockInfoContiguous convertToCompleteBlock() throws IOException {
+ assert getBlockUCState() != BlockUCState.COMPLETE :
+ "Trying to convert a COMPLETE block";
+ return new BlockInfoContiguous(this);
+ }
+
+ /** Set expected locations */
+ public void setExpectedLocations(DatanodeStorageInfo[] targets) {
+ int numLocations = targets == null ? 0 : targets.length;
+ this.replicas = new ArrayList<ReplicaUnderConstruction>(numLocations);
+ for(int i = 0; i < numLocations; i++)
+ replicas.add(
+ new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW));
+ }
+
+ /**
+ * Create array of expected replica locations
+ * (as has been assigned by chooseTargets()).
+ */
+ public DatanodeStorageInfo[] getExpectedStorageLocations() {
+ int numLocations = replicas == null ? 0 : replicas.size();
+ DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations];
+ for(int i = 0; i < numLocations; i++)
+ storages[i] = replicas.get(i).getExpectedStorageLocation();
+ return storages;
+ }
+
+ /** Get the number of expected locations */
+ public int getNumExpectedLocations() {
+ return replicas == null ? 0 : replicas.size();
+ }
+
+ /**
+ * Return the state of the block under construction.
+ * @see BlockUCState
+ */
+ @Override // BlockInfo
+ public BlockUCState getBlockUCState() {
+ return blockUCState;
+ }
+
+ void setBlockUCState(BlockUCState s) {
+ blockUCState = s;
+ }
+
+ /** Get block recovery ID */
+ public long getBlockRecoveryId() {
+ return blockRecoveryId;
+ }
+
+ /** Get recover block */
+ public Block getTruncateBlock() {
+ return truncateBlock;
+ }
+
+ public void setTruncateBlock(Block recoveryBlock) {
+ this.truncateBlock = recoveryBlock;
+ }
+
+ /**
+ * Process the recorded replicas. When about to commit or finish the
+ * pipeline recovery sort out bad replicas.
+ * @param genStamp The final generation stamp for the block.
+ */
+ public void setGenerationStampAndVerifyReplicas(long genStamp) {
+ // Set the generation stamp for the block.
+ setGenerationStamp(genStamp);
+ if (replicas == null)
+ return;
+
+ // Remove the replicas with wrong gen stamp.
+ // The replica list is unchanged.
+ for (ReplicaUnderConstruction r : replicas) {
+ if (genStamp != r.getGenerationStamp()) {
+ r.getExpectedStorageLocation().removeBlock(this);
+ NameNode.blockStateChangeLog.info("BLOCK* Removing stale replica "
+ + "from location: {}", r.getExpectedStorageLocation());
+ }
+ }
+ }
+
+ /**
+ * Commit block's length and generation stamp as reported by the client.
+ * Set block state to {@link BlockUCState#COMMITTED}.
+ * @param block - contains client reported block length and generation
+ * @throws IOException if block ids are inconsistent.
+ */
+ void commitBlock(Block block) throws IOException {
+ if(getBlockId() != block.getBlockId())
+ throw new IOException("Trying to commit inconsistent block: id = "
+ + block.getBlockId() + ", expected id = " + getBlockId());
+ blockUCState = BlockUCState.COMMITTED;
+ this.set(getBlockId(), block.getNumBytes(), block.getGenerationStamp());
+ // Sort out invalid replicas.
+ setGenerationStampAndVerifyReplicas(block.getGenerationStamp());
+ }
+
+ /**
+ * Initialize lease recovery for this block.
+ * Find the first alive data-node starting from the previous primary and
+ * make it primary.
+ */
+ public void initializeBlockRecovery(long recoveryId) {
+ setBlockUCState(BlockUCState.UNDER_RECOVERY);
+ blockRecoveryId = recoveryId;
+ if (replicas.size() == 0) {
+ NameNode.blockStateChangeLog.warn("BLOCK*"
+ + " BlockInfoUnderConstruction.initLeaseRecovery:"
+ + " No blocks found, lease removed.");
+ }
+ boolean allLiveReplicasTriedAsPrimary = true;
+ for (int i = 0; i < replicas.size(); i++) {
+ // Check if all replicas have been tried or not.
+ if (replicas.get(i).isAlive()) {
+ allLiveReplicasTriedAsPrimary =
+ (allLiveReplicasTriedAsPrimary &&
replicas.get(i).getChosenAsPrimary());
+ }
+ }
+ if (allLiveReplicasTriedAsPrimary) {
+ // Just set all the replicas to be chosen whether they are alive or not.
+ for (int i = 0; i < replicas.size(); i++) {
+ replicas.get(i).setChosenAsPrimary(false);
+ }
+ }
+ long mostRecentLastUpdate = 0;
+ ReplicaUnderConstruction primary = null;
+ primaryNodeIndex = -1;
+ for(int i = 0; i < replicas.size(); i++) {
+ // Skip alive replicas which have been chosen for recovery.
+ if (!(replicas.get(i).isAlive() &&
!replicas.get(i).getChosenAsPrimary())) {
+ continue;
+ }
+ final ReplicaUnderConstruction ruc = replicas.get(i);
+ final long lastUpdate =
ruc.getExpectedStorageLocation().getDatanodeDescriptor().getLastUpdate();
+ if (lastUpdate > mostRecentLastUpdate) {
+ primaryNodeIndex = i;
+ primary = ruc;
+ mostRecentLastUpdate = lastUpdate;
+ }
+ }
+ if (primary != null) {
+
primary.getExpectedStorageLocation().getDatanodeDescriptor().addBlockToBeRecovered(this);
+ primary.setChosenAsPrimary(true);
+ NameNode.blockStateChangeLog.info(
+ "BLOCK* {} recovery started, primary={}", this, primary);
+ }
+ }
+
+ void addReplicaIfNotPresent(DatanodeStorageInfo storage,
+ Block block,
+ ReplicaState rState) {
+ Iterator<ReplicaUnderConstruction> it = replicas.iterator();
+ while (it.hasNext()) {
+ ReplicaUnderConstruction r = it.next();
+ DatanodeStorageInfo expectedLocation = r.getExpectedStorageLocation();
+ if(expectedLocation == storage) {
+ // Record the gen stamp from the report
+ r.setGenerationStamp(block.getGenerationStamp());
+ return;
+ } else if (expectedLocation != null &&
+ expectedLocation.getDatanodeDescriptor() ==
+ storage.getDatanodeDescriptor()) {
+
+ // The Datanode reported that the block is on a different storage
+ // than the one chosen by BlockPlacementPolicy. This can occur as
+ // we allow Datanodes to choose the target storage. Update our
+ // state by removing the stale entry and adding a new one.
+ it.remove();
+ break;
+ }
+ }
+ replicas.add(new ReplicaUnderConstruction(block, storage, rState));
+ }
+
+ @Override // BlockInfo
+ // BlockInfoUnderConstruction participates in maps the same way as BlockInfo
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override // BlockInfo
+ public boolean equals(Object obj) {
+ // Sufficient to rely on super's implementation
+ return (this == obj) || super.equals(obj);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder b = new StringBuilder(100);
+ appendStringTo(b);
+ return b.toString();
+ }
+
+ @Override
+ public void appendStringTo(StringBuilder sb) {
+ super.appendStringTo(sb);
+ appendUCParts(sb);
+ }
+
+ private void appendUCParts(StringBuilder sb) {
+ sb.append("{UCState=").append(blockUCState)
+ .append(", primaryNodeIndex=").append(primaryNodeIndex)
+ .append(", replicas=[");
+ if (replicas != null) {
+ Iterator<ReplicaUnderConstruction> iter = replicas.iterator();
+ if (iter.hasNext()) {
+ iter.next().appendStringTo(sb);
+ while (iter.hasNext()) {
+ sb.append(", ");
+ iter.next().appendStringTo(sb);
+ }
+ }
+ }
+ sb.append("]}");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e49b40f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
deleted file mode 100644
index 335fe21..0000000
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-
-/**
- * Represents a block that is currently being constructed.<br>
- * This is usually the last block of a file opened for write or append.
- */
-public class BlockInfoUnderConstruction extends BlockInfo {
- /** Block state. See {@link BlockUCState} */
- private BlockUCState blockUCState;
-
- /**
- * Block replicas as assigned when the block was allocated.
- * This defines the pipeline order.
- */
- private List<ReplicaUnderConstruction> replicas;
-
- /**
- * Index of the primary data node doing the recovery. Useful for log
- * messages.
- */
- private int primaryNodeIndex = -1;
-
- /**
- * The new generation stamp, which this block will have
- * after the recovery succeeds. Also used as a recovery id to identify
- * the right recovery if any of the abandoned recoveries re-appear.
- */
- private long blockRecoveryId = 0;
-
- /**
- * The block source to use in the event of copy-on-write truncate.
- */
- private Block truncateBlock;
-
- /**
- * ReplicaUnderConstruction contains information about replicas while
- * they are under construction.
- * The GS, the length and the state of the replica is as reported by
- * the data-node.
- * It is not guaranteed, but expected, that data-nodes actually have
- * corresponding replicas.
- */
- static class ReplicaUnderConstruction extends Block {
- private final DatanodeStorageInfo expectedLocation;
- private ReplicaState state;
- private boolean chosenAsPrimary;
-
- ReplicaUnderConstruction(Block block,
- DatanodeStorageInfo target,
- ReplicaState state) {
- super(block);
- this.expectedLocation = target;
- this.state = state;
- this.chosenAsPrimary = false;
- }
-
- /**
- * Expected block replica location as assigned when the block was
allocated.
- * This defines the pipeline order.
- * It is not guaranteed, but expected, that the data-node actually has
- * the replica.
- */
- private DatanodeStorageInfo getExpectedStorageLocation() {
- return expectedLocation;
- }
-
- /**
- * Get replica state as reported by the data-node.
- */
- ReplicaState getState() {
- return state;
- }
-
- /**
- * Whether the replica was chosen for recovery.
- */
- boolean getChosenAsPrimary() {
- return chosenAsPrimary;
- }
-
- /**
- * Set replica state.
- */
- void setState(ReplicaState s) {
- state = s;
- }
-
- /**
- * Set whether this replica was chosen for recovery.
- */
- void setChosenAsPrimary(boolean chosenAsPrimary) {
- this.chosenAsPrimary = chosenAsPrimary;
- }
-
- /**
- * Is data-node the replica belongs to alive.
- */
- boolean isAlive() {
- return expectedLocation.getDatanodeDescriptor().isAlive;
- }
-
- @Override // Block
- public int hashCode() {
- return super.hashCode();
- }
-
- @Override // Block
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
- @Override
- public String toString() {
- final StringBuilder b = new StringBuilder(50);
- appendStringTo(b);
- return b.toString();
- }
-
- @Override
- public void appendStringTo(StringBuilder sb) {
- sb.append("ReplicaUC[")
- .append(expectedLocation)
- .append("|")
- .append(state)
- .append("]");
- }
- }
-
- /**
- * Create block and set its state to
- * {@link BlockUCState#UNDER_CONSTRUCTION}.
- */
- public BlockInfoUnderConstruction(Block blk, short replication) {
- this(blk, replication, BlockUCState.UNDER_CONSTRUCTION, null);
- }
-
- /**
- * Create a block that is currently being constructed.
- */
- public BlockInfoUnderConstruction(Block blk, short replication,
- BlockUCState state,
- DatanodeStorageInfo[] targets) {
- super(blk, replication);
- assert getBlockUCState() != BlockUCState.COMPLETE :
- "BlockInfoUnderConstruction cannot be in COMPLETE state";
- this.blockUCState = state;
- setExpectedLocations(targets);
- }
-
- /**
- * Convert an under construction block to a complete block.
- *
- * @return BlockInfo - a complete block.
- * @throws IOException if the state of the block
- * (the generation stamp and the length) has not been committed by
- * the client or it does not have at least a minimal number of replicas
- * reported from data-nodes.
- */
- BlockInfo convertToCompleteBlock() throws IOException {
- assert getBlockUCState() != BlockUCState.COMPLETE :
- "Trying to convert a COMPLETE block";
- return new BlockInfo(this);
- }
-
- /** Set expected locations */
- public void setExpectedLocations(DatanodeStorageInfo[] targets) {
- int numLocations = targets == null ? 0 : targets.length;
- this.replicas = new ArrayList<ReplicaUnderConstruction>(numLocations);
- for(int i = 0; i < numLocations; i++)
- replicas.add(
- new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW));
- }
-
- /**
- * Create array of expected replica locations
- * (as has been assigned by chooseTargets()).
- */
- public DatanodeStorageInfo[] getExpectedStorageLocations() {
- int numLocations = replicas == null ? 0 : replicas.size();
- DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations];
- for(int i = 0; i < numLocations; i++)
- storages[i] = replicas.get(i).getExpectedStorageLocation();
- return storages;
- }
-
- /** Get the number of expected locations */
- public int getNumExpectedLocations() {
- return replicas == null ? 0 : replicas.size();
- }
-
- /**
- * Return the state of the block under construction.
- * @see BlockUCState
- */
- @Override // BlockInfo
- public BlockUCState getBlockUCState() {
- return blockUCState;
- }
-
- void setBlockUCState(BlockUCState s) {
- blockUCState = s;
- }
-
- /** Get block recovery ID */
- public long getBlockRecoveryId() {
- return blockRecoveryId;
- }
-
- /** Get recover block */
- public Block getTruncateBlock() {
- return truncateBlock;
- }
-
- public void setTruncateBlock(Block recoveryBlock) {
- this.truncateBlock = recoveryBlock;
- }
-
- /**
- * Process the recorded replicas. When about to commit or finish the
- * pipeline recovery sort out bad replicas.
- * @param genStamp The final generation stamp for the block.
- */
- public void setGenerationStampAndVerifyReplicas(long genStamp) {
- // Set the generation stamp for the block.
- setGenerationStamp(genStamp);
- if (replicas == null)
- return;
-
- // Remove the replicas with wrong gen stamp.
- // The replica list is unchanged.
- for (ReplicaUnderConstruction r : replicas) {
- if (genStamp != r.getGenerationStamp()) {
- r.getExpectedStorageLocation().removeBlock(this);
- NameNode.blockStateChangeLog.info("BLOCK* Removing stale replica "
- + "from location: {}", r.getExpectedStorageLocation());
- }
- }
- }
-
- /**
- * Commit block's length and generation stamp as reported by the client.
- * Set block state to {@link BlockUCState#COMMITTED}.
- * @param block - contains client reported block length and generation
- * @throws IOException if block ids are inconsistent.
- */
- void commitBlock(Block block) throws IOException {
- if(getBlockId() != block.getBlockId())
- throw new IOException("Trying to commit inconsistent block: id = "
- + block.getBlockId() + ", expected id = " + getBlockId());
- blockUCState = BlockUCState.COMMITTED;
- this.set(getBlockId(), block.getNumBytes(), block.getGenerationStamp());
- // Sort out invalid replicas.
- setGenerationStampAndVerifyReplicas(block.getGenerationStamp());
- }
-
- /**
- * Initialize lease recovery for this block.
- * Find the first alive data-node starting from the previous primary and
- * make it primary.
- */
- public void initializeBlockRecovery(long recoveryId) {
- setBlockUCState(BlockUCState.UNDER_RECOVERY);
- blockRecoveryId = recoveryId;
- if (replicas.size() == 0) {
- NameNode.blockStateChangeLog.warn("BLOCK*"
- + " BlockInfoUnderConstruction.initLeaseRecovery:"
- + " No blocks found, lease removed.");
- }
- boolean allLiveReplicasTriedAsPrimary = true;
- for (int i = 0; i < replicas.size(); i++) {
- // Check if all replicas have been tried or not.
- if (replicas.get(i).isAlive()) {
- allLiveReplicasTriedAsPrimary =
- (allLiveReplicasTriedAsPrimary &&
replicas.get(i).getChosenAsPrimary());
- }
- }
- if (allLiveReplicasTriedAsPrimary) {
- // Just set all the replicas to be chosen whether they are alive or not.
- for (int i = 0; i < replicas.size(); i++) {
- replicas.get(i).setChosenAsPrimary(false);
- }
- }
- long mostRecentLastUpdate = 0;
- ReplicaUnderConstruction primary = null;
- primaryNodeIndex = -1;
- for(int i = 0; i < replicas.size(); i++) {
- // Skip alive replicas which have been chosen for recovery.
- if (!(replicas.get(i).isAlive() &&
!replicas.get(i).getChosenAsPrimary())) {
- continue;
- }
- final ReplicaUnderConstruction ruc = replicas.get(i);
- final long lastUpdate =
ruc.getExpectedStorageLocation().getDatanodeDescriptor().getLastUpdate();
- if (lastUpdate > mostRecentLastUpdate) {
- primaryNodeIndex = i;
- primary = ruc;
- mostRecentLastUpdate = lastUpdate;
- }
- }
- if (primary != null) {
-
primary.getExpectedStorageLocation().getDatanodeDescriptor().addBlockToBeRecovered(this);
- primary.setChosenAsPrimary(true);
- NameNode.blockStateChangeLog.info(
- "BLOCK* {} recovery started, primary={}", this, primary);
- }
- }
-
- void addReplicaIfNotPresent(DatanodeStorageInfo storage,
- Block block,
- ReplicaState rState) {
- Iterator<ReplicaUnderConstruction> it = replicas.iterator();
- while (it.hasNext()) {
- ReplicaUnderConstruction r = it.next();
- DatanodeStorageInfo expectedLocation = r.getExpectedStorageLocation();
- if(expectedLocation == storage) {
- // Record the gen stamp from the report
- r.setGenerationStamp(block.getGenerationStamp());
- return;
- } else if (expectedLocation != null &&
- expectedLocation.getDatanodeDescriptor() ==
- storage.getDatanodeDescriptor()) {
-
- // The Datanode reported that the block is on a different storage
- // than the one chosen by BlockPlacementPolicy. This can occur as
- // we allow Datanodes to choose the target storage. Update our
- // state by removing the stale entry and adding a new one.
- it.remove();
- break;
- }
- }
- replicas.add(new ReplicaUnderConstruction(block, storage, rState));
- }
-
- @Override // BlockInfo
- // BlockInfoUnderConstruction participates in maps the same way as BlockInfo
- public int hashCode() {
- return super.hashCode();
- }
-
- @Override // BlockInfo
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
- @Override
- public String toString() {
- final StringBuilder b = new StringBuilder(100);
- appendStringTo(b);
- return b.toString();
- }
-
- @Override
- public void appendStringTo(StringBuilder sb) {
- super.appendStringTo(sb);
- appendUCParts(sb);
- }
-
- private void appendUCParts(StringBuilder sb) {
- sb.append("{UCState=").append(blockUCState)
- .append(", primaryNodeIndex=").append(primaryNodeIndex)
- .append(", replicas=[");
- if (replicas != null) {
- Iterator<ReplicaUnderConstruction> iter = replicas.iterator();
- if (iter.hasNext()) {
- iter.next().appendStringTo(sb);
- while (iter.hasNext()) {
- sb.append(", ");
- iter.next().appendStringTo(sb);
- }
- }
- }
- sb.append("]}");
- }
-}
