Merge branch 'trunk' into HDFS-7240
Conflicts:
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b14a70e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b14a70e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b14a70e7
Branch: refs/heads/HDFS-7240
Commit: b14a70e792f8f5d6ebc83d6ce1e9bf8e3668fcff
Parents: 28d313d 6c17d31
Author: Anu Engineer <[email protected]>
Authored: Wed Sep 30 11:19:14 2015 -0700
Committer: Anu Engineer <[email protected]>
Committed: Wed Sep 30 11:19:14 2015 -0700
----------------------------------------------------------------------
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 74 ++
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/fs/CommonConfigurationKeys.java | 15 +
.../org/apache/hadoop/fs/FSOutputSummer.java | 7 +-
.../main/java/org/apache/hadoop/fs/FsShell.java | 8 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 144 +++
.../apache/hadoop/io/erasurecode/ECBlock.java | 80 ++
.../hadoop/io/erasurecode/ECBlockGroup.java | 100 ++
.../apache/hadoop/io/erasurecode/ECChunk.java | 87 ++
.../apache/hadoop/io/erasurecode/ECSchema.java | 227 ++++
.../erasurecode/codec/AbstractErasureCodec.java | 51 +
.../io/erasurecode/codec/ErasureCodec.java | 49 +
.../io/erasurecode/codec/RSErasureCodec.java | 43 +
.../io/erasurecode/codec/XORErasureCodec.java | 44 +
.../erasurecode/coder/AbstractErasureCoder.java | 62 ++
.../coder/AbstractErasureCodingStep.java | 59 +
.../coder/AbstractErasureDecoder.java | 167 +++
.../coder/AbstractErasureEncoder.java | 60 +
.../io/erasurecode/coder/ErasureCoder.java | 77 ++
.../io/erasurecode/coder/ErasureCodingStep.java | 55 +
.../erasurecode/coder/ErasureDecodingStep.java | 52 +
.../erasurecode/coder/ErasureEncodingStep.java | 49 +
.../io/erasurecode/coder/RSErasureDecoder.java | 67 ++
.../io/erasurecode/coder/RSErasureEncoder.java | 67 ++
.../io/erasurecode/coder/XORErasureDecoder.java | 86 ++
.../io/erasurecode/coder/XORErasureEncoder.java | 53 +
.../io/erasurecode/grouper/BlockGrouper.java | 90 ++
.../rawcoder/AbstractRawErasureCoder.java | 138 +++
.../rawcoder/AbstractRawErasureDecoder.java | 207 ++++
.../rawcoder/AbstractRawErasureEncoder.java | 136 +++
.../io/erasurecode/rawcoder/RSRawDecoder.java | 216 ++++
.../io/erasurecode/rawcoder/RSRawEncoder.java | 79 ++
.../rawcoder/RSRawErasureCoderFactory.java | 34 +
.../erasurecode/rawcoder/RawErasureCoder.java | 66 ++
.../rawcoder/RawErasureCoderFactory.java | 42 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 88 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 64 ++
.../io/erasurecode/rawcoder/XORRawDecoder.java | 83 ++
.../io/erasurecode/rawcoder/XORRawEncoder.java | 77 ++
.../rawcoder/XORRawErasureCoderFactory.java | 34 +
.../io/erasurecode/rawcoder/util/DumpUtil.java | 85 ++
.../erasurecode/rawcoder/util/GaloisField.java | 561 ++++++++++
.../io/erasurecode/rawcoder/util/RSUtil.java | 39 +
.../apache/hadoop/fs/shell/TestTextCommand.java | 15 +-
.../hadoop/io/erasurecode/BufferAllocator.java | 91 ++
.../hadoop/io/erasurecode/TestCoderBase.java | 500 +++++++++
.../hadoop/io/erasurecode/TestECSchema.java | 49 +
.../erasurecode/coder/TestErasureCoderBase.java | 297 +++++
.../erasurecode/coder/TestRSErasureCoder.java | 126 +++
.../io/erasurecode/coder/TestXORCoder.java | 64 ++
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 118 ++
.../rawcoder/TestRSRawCoderBase.java | 58 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 232 ++++
.../erasurecode/rawcoder/TestXORRawCoder.java | 66 ++
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 +
.../org/apache/hadoop/hdfs/BlockReader.java | 10 +-
.../apache/hadoop/hdfs/BlockReaderFactory.java | 114 +-
.../apache/hadoop/hdfs/BlockReaderLocal.java | 82 +-
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 25 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 145 ++-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 169 +--
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 151 ++-
.../java/org/apache/hadoop/hdfs/DFSPacket.java | 34 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 972 ++++++++++++++++
.../hadoop/hdfs/DFSStripedOutputStream.java | 953 ++++++++++++++++
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 18 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 295 +++--
.../hadoop/hdfs/DistributedFileSystem.java | 76 ++
.../apache/hadoop/hdfs/ExternalBlockReader.java | 6 +
.../apache/hadoop/hdfs/RemoteBlockReader.java | 9 +-
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 30 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 187 ++++
.../hdfs/client/HdfsClientConfigKeys.java | 12 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 17 +
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 18 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 27 +
.../apache/hadoop/hdfs/protocol/DatanodeID.java | 2 +
.../hdfs/protocol/ErasureCodingPolicy.java | 93 ++
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 9 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 5 +-
.../hadoop/hdfs/protocol/LocatedBlock.java | 8 +-
.../hadoop/hdfs/protocol/LocatedBlocks.java | 15 +-
.../hdfs/protocol/LocatedStripedBlock.java | 86 ++
.../protocol/SnapshottableDirectoryStatus.java | 2 +-
.../protocol/datatransfer/PacketReceiver.java | 8 +-
.../hdfs/protocol/datatransfer/Sender.java | 6 +-
.../datatransfer/sasl/DataTransferSaslUtil.java | 8 +-
.../ClientDatanodeProtocolTranslatorPB.java | 10 +-
.../ClientNamenodeProtocolTranslatorPB.java | 76 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 164 ++-
.../hdfs/shortcircuit/DfsClientShmManager.java | 45 +-
.../hdfs/shortcircuit/ShortCircuitCache.java | 101 +-
.../hdfs/shortcircuit/ShortCircuitReplica.java | 33 +-
.../hdfs/shortcircuit/ShortCircuitShm.java | 14 +-
.../hadoop/hdfs/util/ByteArrayManager.java | 63 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 952 ++++++++++++++++
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 4 +-
.../org/apache/hadoop/hdfs/web/TokenAspect.java | 8 +-
.../hadoop/hdfs/web/URLConnectionFactory.java | 4 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 7 +
.../src/main/proto/erasurecoding.proto | 60 +
.../src/main/proto/hdfs.proto | 41 +
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 455 ++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 +
.../hadoop-hdfs/src/main/bin/hdfs | 6 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 21 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 40 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 36 +
...tNamenodeProtocolServerSideTranslatorPB.java | 65 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 2 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 2 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 185 +++-
.../hadoop/hdfs/server/balancer/Balancer.java | 6 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 150 ++-
.../server/blockmanagement/BlockCollection.java | 20 +-
.../server/blockmanagement/BlockIdManager.java | 58 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 99 +-
.../blockmanagement/BlockInfoContiguous.java | 21 +-
.../blockmanagement/BlockInfoStriped.java | 236 ++++
.../server/blockmanagement/BlockManager.java | 1039 ++++++++++++------
.../blockmanagement/BlockPlacementPolicies.java | 54 +
.../blockmanagement/BlockPlacementPolicy.java | 26 +-
.../blockmanagement/BlockRecoveryWork.java | 111 ++
.../blockmanagement/BlockToMarkCorrupt.java | 19 +-
.../BlockUnderConstructionFeature.java | 46 +-
.../hdfs/server/blockmanagement/BlocksMap.java | 27 +-
.../blockmanagement/DatanodeDescriptor.java | 48 +-
.../server/blockmanagement/DatanodeManager.java | 18 +-
.../blockmanagement/DatanodeStorageInfo.java | 25 +-
.../blockmanagement/DecommissionManager.java | 78 +-
.../blockmanagement/ErasureCodingWork.java | 60 +
.../server/blockmanagement/ReplicationWork.java | 64 +-
.../SequentialBlockGroupIdGenerator.java | 86 ++
.../SequentialBlockIdGenerator.java | 6 +-
.../blockmanagement/UnderReplicatedBlocks.java | 51 +-
.../hdfs/server/common/HdfsServerConstants.java | 5 +
.../hdfs/server/datanode/BPOfferService.java | 8 +
.../hadoop/hdfs/server/datanode/DNConf.java | 27 +
.../hadoop/hdfs/server/datanode/DataNode.java | 57 +-
.../hdfs/server/datanode/DirectoryScanner.java | 8 -
.../hdfs/server/datanode/StorageLocation.java | 2 +-
.../erasurecode/ErasureCodingWorker.java | 1016 +++++++++++++++++
.../server/datanode/web/DatanodeHttpServer.java | 15 +
.../apache/hadoop/hdfs/server/mover/Mover.java | 37 +-
.../namenode/ErasureCodingPolicyManager.java | 115 ++
.../hdfs/server/namenode/FSDirAppendOp.java | 7 +
.../hdfs/server/namenode/FSDirAttrOp.java | 9 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 6 +
.../server/namenode/FSDirErasureCodingOp.java | 277 +++++
.../server/namenode/FSDirStatAndListingOp.java | 22 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 13 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 175 +--
.../hdfs/server/namenode/FSDirectory.java | 2 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 89 +-
.../hdfs/server/namenode/FSImageFormat.java | 31 +-
.../server/namenode/FSImageFormatPBINode.java | 42 +-
.../server/namenode/FSImageFormatProtobuf.java | 9 +-
.../server/namenode/FSImageSerialization.java | 22 +-
.../hdfs/server/namenode/FSNamesystem.java | 251 ++++-
.../hadoop/hdfs/server/namenode/INodeFile.java | 145 ++-
.../server/namenode/INodeFileAttributes.java | 22 +-
.../hdfs/server/namenode/LeaseManager.java | 2 +-
.../server/namenode/NameNodeLayoutVersion.java | 3 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 38 +-
.../hdfs/server/namenode/NamenodeFsck.java | 228 +++-
.../hadoop/hdfs/server/namenode/Namesystem.java | 16 +-
.../hadoop/hdfs/server/namenode/SafeMode.java | 5 +-
.../snapshot/FSImageFormatPBSnapshot.java | 9 +-
.../server/namenode/snapshot/FileDiffList.java | 9 +-
.../server/protocol/BlockECRecoveryCommand.java | 146 +++
.../server/protocol/BlocksWithLocations.java | 30 +
.../hdfs/server/protocol/DatanodeProtocol.java | 1 +
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 48 +
.../hdfs/tools/erasurecode/ECCommand.java | 214 ++++
.../StorageContainerNameService.java | 13 +-
.../src/main/proto/DatanodeProtocol.proto | 10 +
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 11 +-
.../hadoop-hdfs/src/main/proto/fsimage.proto | 2 +
.../src/main/resources/hdfs-default.xml | 31 +-
.../src/site/markdown/HDFSCommands.md | 20 +
.../src/site/markdown/HDFSErasureCoding.md | 135 +++
.../hadoop/cli/CLITestCmdErasureCoding.java | 39 +
.../apache/hadoop/cli/TestErasureCodingCLI.java | 115 ++
.../cli/util/CLICommandErasureCodingCli.java | 21 +
.../cli/util/ErasureCodingCliCmdExecutor.java | 37 +
.../apache/hadoop/hdfs/BlockReaderTestUtil.java | 7 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 162 ++-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 491 +++++++++
.../hadoop/hdfs/TestBlockReaderFactory.java | 16 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 6 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 335 ++++++
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 156 +++
.../TestDFSStripedOutputStreamWithFailure.java | 473 ++++++++
...estDFSStripedOutputStreamWithFailure000.java | 22 +
...estDFSStripedOutputStreamWithFailure010.java | 22 +
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 146 ++-
.../apache/hadoop/hdfs/TestDatanodeConfig.java | 4 +-
.../apache/hadoop/hdfs/TestDecommission.java | 3 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 249 +++++
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 65 ++
.../java/org/apache/hadoop/hdfs/TestLease.java | 4 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 362 ++++++
.../TestReadStripedFileWithMissingBlocks.java | 151 +++
.../hadoop/hdfs/TestRecoverStripedFile.java | 418 +++++++
.../org/apache/hadoop/hdfs/TestSafeMode.java | 4 +-
.../hdfs/TestSafeModeWithStripedFile.java | 154 +++
.../hadoop/hdfs/TestWriteReadStripedFile.java | 251 +++++
.../hdfs/TestWriteStripedFileWithFailure.java | 171 +++
.../hadoop/hdfs/protocol/TestLayoutVersion.java | 3 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 170 ++-
.../hdfs/server/balancer/TestBalancer.java | 98 +-
.../blockmanagement/BlockManagerTestUtil.java | 10 +-
.../server/blockmanagement/TestBlockInfo.java | 14 +-
.../blockmanagement/TestBlockInfoStriped.java | 220 ++++
.../blockmanagement/TestBlockManager.java | 63 +-
.../blockmanagement/TestBlockTokenWithDFS.java | 422 +++----
.../TestBlockTokenWithDFSStriped.java | 115 ++
.../TestNameNodePrunesMissingStorages.java | 5 +-
.../server/blockmanagement/TestNodeCount.java | 2 +-
.../TestOverReplicatedBlocks.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 20 +-
.../TestSequentialBlockGroupId.java | 222 ++++
.../TestUnderReplicatedBlockQueues.java | 59 +
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../datanode/TestIncrementalBrVariations.java | 14 +-
.../hadoop/hdfs/server/mover/TestMover.java | 123 ++-
.../hdfs/server/namenode/NameNodeAdapter.java | 9 +-
.../TestAddOverReplicatedStripedBlocks.java | 265 +++++
.../server/namenode/TestAddStripedBlocks.java | 430 ++++++++
.../hdfs/server/namenode/TestDeadDatanode.java | 3 +-
.../server/namenode/TestFSEditLogLoader.java | 269 +++++
.../hdfs/server/namenode/TestFSImage.java | 346 +++++-
.../hdfs/server/namenode/TestFileTruncate.java | 4 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 83 +-
.../hdfs/server/namenode/TestINodeFile.java | 2 +-
.../namenode/TestQuotaWithStripedBlocks.java | 125 +++
.../namenode/TestRecoverStripedBlocks.java | 169 +++
.../server/namenode/TestStripedINodeFile.java | 285 +++++
.../namenode/ha/TestRetryCacheWithHA.java | 9 +-
.../namenode/snapshot/SnapshotTestHelper.java | 2 +-
...TestOfflineImageViewerWithStripedBlocks.java | 162 +++
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 279 +++++
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 377 +++++++
hadoop-project/src/site/site.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/LocalResourcesTracker.java | 3 +-
.../localizer/LocalResourcesTrackerImpl.java | 24 +-
.../localizer/ResourceLocalizationService.java | 5 +-
.../TestLocalResourcesTrackerImpl.java | 65 +-
.../TestResourceLocalizationService.java | 16 +-
254 files changed, 23374 insertions(+), 2218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 88b4594,d35fb57..5c22be6
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@@ -444,3 -478,11 +478,10 @@@ message RollingUpgradeStatusProto
required string blockPoolId = 1;
optional bool finalized = 2 [default = false];
}
+
-
+ /**
+ * A list of storage IDs.
+ */
+ message StorageUuidsProto {
+ repeated string storageUuids = 1;
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index e039b3b,18bfc41..41b572f
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -282,15 -293,11 +294,18 @@@ public class BlockManager implements Bl
/** Check whether name system is running before terminating */
private boolean checkNSRunning = true;
+ /** Check whether there are any non-EC blocks using StripedID */
+ private boolean hasNonEcBlockUsingStripedID = false;
+
public BlockManager(final Namesystem namesystem, final Configuration conf)
- throws IOException {
+ throws IOException {
+ // Compute the map capacity by allocating 2% of total memory
+ this(namesystem, conf, new BlocksMap(
+ LightWeightGSet.computeCapacity(2.0, "BlocksMap")));
+ }
+
+ public BlockManager(final Namesystem namesystem, final Configuration conf,
+ BlocksMap blocksMap) throws IOException {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
@@@ -301,8 -308,10 +316,8 @@@
invalidateBlocks = new InvalidateBlocks(
datanodeManager.blockInvalidateLimit, startupDelayBlockDeletionInMs);
- // Compute the map capacity by allocating 2% of total memory
- blocksMap = new BlocksMap(
- LightWeightGSet.computeCapacity(2.0, "BlocksMap"));
+ this.blocksMap = blocksMap;
- blockplacement = BlockPlacementPolicy.getInstance(
+ placementPolicies = new BlockPlacementPolicies(
conf, datanodeManager.getFSClusterStats(),
datanodeManager.getNetworkTopology(),
datanodeManager.getHost2DatanodeMap());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 8bd8c51,bb9a706..85e167d
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@@ -17,16 -17,18 +17,19 @@@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+ import java.util.Arrays;
+ import java.util.Iterator;
+ import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
++
import org.apache.hadoop.fs.StorageType;
+ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
- import java.util.Arrays;
- import java.util.Iterator;
- import java.util.List;
-import com.google.common.annotations.VisibleForTesting;
/**
* A Datanode has one or more storages. A storage in the Datanode is
represented
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index a6c8fda,96e74e5..c7eb468
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@@ -31,8 -31,8 +31,9 @@@ import org.apache.hadoop.hdfs.protocol.
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.datanode.dataset.DatasetSpi;
import org.apache.hadoop.hdfs.server.protocol.*;
+ import
org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import
org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 6bd556f,10a8a9c..da09b0e
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@@ -156,14 -155,11 +157,15 @@@ import org.apache.hadoop.hdfs.server.co
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.datanode.DataStorage.VolumeBuilder;
import
org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
+import org.apache.hadoop.hdfs.server.datanode.dataset.DatasetSpi;
+ import org.apache.hadoop.hdfs.server.datanode.erasurecode.ErasureCodingWorker;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.dataset.VolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
import org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer;
import
org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
index 60e9c20,0000000..13cff36
mode 100644,000000..100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
@@@ -1,170 -1,0 +1,177 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.storagecontainer;
+
+import org.apache.hadoop.hdfs.protocol.Block;
++import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.namenode.CacheManager;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.AccessControlException;
+
++import java.io.IOException;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Namesystem implementation to be used by StorageContainerManager.
+ */
+public class StorageContainerNameService implements Namesystem {
+
+ private ReentrantReadWriteLock coarseLock = new ReentrantReadWriteLock();
+ private String blockPoolId;
+ private volatile boolean serviceRunning = true;
+
+ public void shutdown() {
+ serviceRunning = false;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return serviceRunning;
+ }
+
+ @Override
+ public void checkSuperuserPrivilege() throws AccessControlException {
+ // TBD
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ return blockPoolId;
+ }
+
+ public void setBlockPoolId(String id) {
+ this.blockPoolId = id;
+ }
+
+ @Override
+ public boolean isInStandbyState() {
+ // HA mode is not supported
+ return false;
+ }
+
+ @Override
+ public boolean isGenStampInFuture(Block block) {
+ // HA mode is not supported
+ return false;
+ }
+
+ @Override
+ public BlockCollection getBlockCollection(long id) {
+ return null;
+ }
+
+ @Override
+ public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal) {
+ // TBD
+ }
+
+ @Override
+ public void checkOperation(NameNode.OperationCategory read)
+ throws StandbyException {
+ // HA mode is not supported
+ }
+
+ @Override
++ public ErasureCodingPolicy getErasureCodingPolicyForPath(String src)
++ throws IOException {
++ return null;
++ }
++
++ @Override
+ public boolean isInSnapshot(BlockInfo blockInfo) {
+ // Snapshots not supported
+ return false;
+ }
+
+ @Override
+ public CacheManager getCacheManager() {
+ // Cache Management is not supported
+ return null;
+ }
+
+ @Override
+ public HAContext getHAContext() {
+ return null;
+ }
+
+ @Override
+ public void readLock() {
+ coarseLock.readLock().lock();
+ }
+
+ @Override
+ public void readUnlock() {
+ coarseLock.readLock().unlock();
+ }
+
+ @Override
+ public boolean hasReadLock() {
+ return coarseLock.getReadHoldCount() > 0 || hasWriteLock();
+ }
+
+ @Override
+ public void writeLock() {
+ coarseLock.writeLock().lock();
+ }
+
+ @Override
+ public void writeLockInterruptibly() throws InterruptedException {
+ coarseLock.writeLock().lockInterruptibly();
+ }
+
+ @Override
+ public void writeUnlock() {
+ coarseLock.writeLock().unlock();
+ }
+
+ @Override
+ public boolean hasWriteLock() {
+ return coarseLock.isWriteLockedByCurrentThread();
+ }
+
+ @Override
+ public void checkSafeMode() {
+ // TBD
+ }
+
+ @Override
+ public boolean isInSafeMode() {
+ return false;
+ }
+
+ @Override
+ public boolean isInStartupSafeMode() {
+ return false;
+ }
+
-
+ @Override
- public void incrementSafeBlockCount(int replication) {
- // Do nothing
++ public void incrementSafeBlockCount(int replication, BlockInfo storedBlock)
{
++ // Do nothing
+ }
+
+ @Override
+ public void decrementSafeBlockCount(BlockInfo b) {
+ // Do nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b14a70e7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
