Repository: hadoop Updated Branches: refs/heads/trunk 10cf5773b -> 5a7ca6ac3
HDDS-837. Persist originNodeId as part of .container file in datanode. Contributed by Nanda kumar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a7ca6ac Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a7ca6ac Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a7ca6ac Branch: refs/heads/trunk Commit: 5a7ca6ac3e9964c1fbd5bb654d39d5b8fb731701 Parents: 10cf577 Author: Nanda kumar <na...@apache.org> Authored: Mon Nov 19 14:38:45 2018 +0530 Committer: Nanda kumar <na...@apache.org> Committed: Mon Nov 19 14:39:55 2018 +0530 ---------------------------------------------------------------------- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../main/proto/DatanodeContainerProtocol.proto | 41 +++++++-------- .../container/common/impl/ContainerData.java | 44 +++++++++++++++-- .../common/impl/ContainerDataYaml.java | 7 ++- .../container/common/impl/HddsDispatcher.java | 1 + .../container/common/interfaces/Handler.java | 12 +++++ .../server/ratis/ContainerStateMachine.java | 7 ++- .../container/keyvalue/KeyValueContainer.java | 3 +- .../keyvalue/KeyValueContainerData.java | 52 +++----------------- .../container/keyvalue/KeyValueHandler.java | 7 ++- .../ozoneimpl/ContainerController.java | 9 ++-- .../DownloadAndImportReplicator.java | 2 + .../StorageContainerDatanodeProtocol.proto | 1 + .../common/TestKeyValueContainerData.java | 7 ++- .../common/impl/TestContainerDataYaml.java | 4 +- .../container/common/impl/TestContainerSet.java | 7 ++- .../common/impl/TestHddsDispatcher.java | 20 ++++++-- .../common/interfaces/TestHandler.java | 11 ++++- .../TestCloseContainerCommandHandler.java | 29 +++++------ .../keyvalue/TestBlockManagerImpl.java | 7 +-- .../keyvalue/TestChunkManagerImpl.java | 6 ++- .../keyvalue/TestKeyValueBlockIterator.java | 3 +- .../keyvalue/TestKeyValueContainer.java | 13 +++-- .../container/keyvalue/TestKeyValueHandler.java | 17 +++++-- .../keyvalue/TestTarContainerPacker.java | 4 +- .../container/ozoneimpl/TestOzoneContainer.java | 14 +++++- .../replication/TestReplicationSupervisor.java | 4 +- .../test/resources/additionalfields.container | 4 +- .../test/resources/incorrect.checksum.container | 2 + .../src/test/resources/incorrect.container | 2 + .../ozone/container/common/TestEndPoint.java | 23 +++++++-- .../common/TestBlockDeletingService.java | 3 +- .../TestContainerDeletionChoosingPolicy.java | 8 ++- .../common/impl/TestContainerPersistence.java | 3 +- .../container/metrics/TestContainerMetrics.java | 15 ++++-- .../container/ozoneimpl/TestOzoneContainer.java | 10 +++- 36 files changed, 266 insertions(+), 138 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index d37ce0f..f8f31c8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -188,6 +188,8 @@ public final class OzoneConsts { public static final String CHUNKS_PATH = "chunksPath"; public static final String CONTAINER_DB_TYPE = "containerDBType"; public static final String CHECKSUM = "checksum"; + public static final String ORIGIN_PIPELINE_ID = "originPipelineId"; + public static final String ORIGIN_NODE_ID = "originNodeId"; // For OM Audit usage public static final String VOLUME = "volume"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto index ac2d277..9e94dd1 100644 --- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto @@ -168,31 +168,32 @@ message ContainerCommandRequestProto { required int64 containerID = 3; required string datanodeUuid = 4; + optional string pipelineID = 5; // One of the following command is available when the corresponding // cmdType is set. At the protocol level we allow only // one command in each packet. // TODO : Upgrade to Protobuf 2.6 or later. - optional CreateContainerRequestProto createContainer = 5; - optional ReadContainerRequestProto readContainer = 6; - optional UpdateContainerRequestProto updateContainer = 7; - optional DeleteContainerRequestProto deleteContainer = 8; - optional ListContainerRequestProto listContainer = 9; - optional CloseContainerRequestProto closeContainer = 10; - - optional PutBlockRequestProto putBlock = 11; - optional GetBlockRequestProto getBlock = 12; - optional DeleteBlockRequestProto deleteBlock = 13; - optional ListBlockRequestProto listBlock = 14; - - optional ReadChunkRequestProto readChunk = 15; - optional WriteChunkRequestProto writeChunk = 16; - optional DeleteChunkRequestProto deleteChunk = 17; - optional ListChunkRequestProto listChunk = 18; - - optional PutSmallFileRequestProto putSmallFile = 19; - optional GetSmallFileRequestProto getSmallFile = 20; - optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 21; + optional CreateContainerRequestProto createContainer = 6; + optional ReadContainerRequestProto readContainer = 7; + optional UpdateContainerRequestProto updateContainer = 8; + optional DeleteContainerRequestProto deleteContainer = 9; + optional ListContainerRequestProto listContainer = 10; + optional CloseContainerRequestProto closeContainer = 11; + + optional PutBlockRequestProto putBlock = 12; + optional GetBlockRequestProto getBlock = 13; + optional DeleteBlockRequestProto deleteBlock = 14; + optional ListBlockRequestProto listBlock = 15; + + optional ReadChunkRequestProto readChunk = 16; + optional WriteChunkRequestProto writeChunk = 17; + optional DeleteChunkRequestProto deleteChunk = 18; + optional ListChunkRequestProto listChunk = 19; + + optional PutSmallFileRequestProto putSmallFile = 20; + optional GetSmallFileRequestProto getSmallFile = 21; + optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 22; } message ContainerCommandResponseProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java index 6f51517..d2fa2c8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java @@ -42,6 +42,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.LAYOUTVERSION; import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE; import static org.apache.hadoop.ozone.OzoneConsts.METADATA; +import static org.apache.hadoop.ozone.OzoneConsts.ORIGIN_NODE_ID; +import static org.apache.hadoop.ozone.OzoneConsts.ORIGIN_PIPELINE_ID; import static org.apache.hadoop.ozone.OzoneConsts.STATE; /** @@ -69,6 +71,11 @@ public abstract class ContainerData { private final long maxSize; + //ID of the pipeline where this container is created + private String originPipelineId; + //ID of the datanode where this container is created + private String originNodeId; + /** parameters for read/write statistics on the container. **/ private final AtomicLong readBytes; private final AtomicLong writeBytes; @@ -93,17 +100,22 @@ public abstract class ContainerData { STATE, METADATA, MAX_SIZE, - CHECKSUM)); + CHECKSUM, + ORIGIN_PIPELINE_ID, + ORIGIN_NODE_ID)); /** * Creates a ContainerData Object, which holds metadata of the container. * @param type - ContainerType * @param containerId - ContainerId * @param size - container maximum size in bytes + * @param originPipelineId - Pipeline Id where this container is/was created + * @param originNodeId - Node Id where this container is/was created */ - protected ContainerData(ContainerType type, long containerId, long size) { - this(type, containerId, - ChunkLayOutVersion.getLatestVersion().getVersion(), size); + protected ContainerData(ContainerType type, long containerId, long size, + String originPipelineId, String originNodeId) { + this(type, containerId, ChunkLayOutVersion.getLatestVersion().getVersion(), + size, originPipelineId, originNodeId); } /** @@ -112,9 +124,12 @@ public abstract class ContainerData { * @param containerId - ContainerId * @param layOutVersion - Container layOutVersion * @param size - Container maximum size in bytes + * @param originPipelineId - Pipeline Id where this container is/was created + * @param originNodeId - Node Id where this container is/was created */ protected ContainerData(ContainerType type, long containerId, - int layOutVersion, long size) { + int layOutVersion, long size, String originPipelineId, + String originNodeId) { Preconditions.checkNotNull(type); this.containerType = type; @@ -129,6 +144,8 @@ public abstract class ContainerData { this.bytesUsed = new AtomicLong(0L); this.keyCount = new AtomicLong(0L); this.maxSize = size; + this.originPipelineId = originPipelineId; + this.originNodeId = originNodeId; setChecksumTo0ByteArray(); } @@ -418,6 +435,23 @@ public abstract class ContainerData { return this.checksum; } + + /** + * Returns the origin pipeline Id of this container. + * @return origin node Id + */ + public String getOriginPipelineId() { + return originPipelineId; + } + + /** + * Returns the origin node Id of this container. + * @return origin node Id + */ + public String getOriginNodeId() { + return originNodeId; + } + /** * Compute the checksum for ContainerData using the specified Yaml (based * on ContainerType) and set the checksum. http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java index af705c4..2d09124 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java @@ -238,9 +238,14 @@ public final class ContainerDataYaml { long size = (long) nodes.get(OzoneConsts.MAX_SIZE); + String originPipelineId = (String) nodes.get( + OzoneConsts.ORIGIN_PIPELINE_ID); + String originNodeId = (String) nodes.get(OzoneConsts.ORIGIN_NODE_ID); + //When a new field is added, it needs to be added here. KeyValueContainerData kvData = new KeyValueContainerData( - (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size); + (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size, + originPipelineId, originNodeId); kvData.setContainerDBType((String)nodes.get( OzoneConsts.CONTAINER_DB_TYPE)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index ec58aa5..b8669fb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -262,6 +262,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { .setCmdType(ContainerProtos.Type.CreateContainer) .setContainerID(containerRequest.getContainerID()) .setCreateContainer(createRequest.build()) + .setPipelineID(containerRequest.getPipelineID()) .setDatanodeUuid(containerRequest.getDatanodeUuid()) .setTraceID(containerRequest.getTraceID()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java index 19e61c4..ad55618 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java @@ -23,6 +23,7 @@ import java.io.FileInputStream; import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -52,6 +53,7 @@ public abstract class Handler { protected final ContainerMetrics metrics; private final StateContext context; + private final DatanodeDetails datanodeDetails; protected Handler(Configuration config, StateContext context, ContainerSet contSet, VolumeSet volumeSet, @@ -61,6 +63,7 @@ public abstract class Handler { this.containerSet = contSet; this.volumeSet = volumeSet; this.metrics = containerMetrics; + this.datanodeDetails = context.getParent().getDatanodeDetails(); } public static Handler getHandlerForContainerType( @@ -77,6 +80,13 @@ public abstract class Handler { } /** + * Returns the Id of this datanode. + * @return datanode Id + */ + protected DatanodeDetails getDatanodeDetails() { + return datanodeDetails; + } + /** * This should be called whenever there is state change. It will trigger * an ICR to SCM. * @@ -101,6 +111,8 @@ public abstract class Handler { public abstract Container importContainer( long containerID, long maxSize, + String originPipelineId, + String originNodeId, FileInputStream rawContainerStream, TarContainerPacker packer) throws IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index a3b496a..24ecbb2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -281,7 +281,12 @@ public class ContainerStateMachine extends BaseStateMachine { private ContainerCommandRequestProto getRequestProto(ByteString request) throws InvalidProtocolBufferException { - return ContainerCommandRequestProto.parseFrom(request); + // TODO: We can avoid creating new builder and set pipeline Id if + // the client is already sending the pipeline id, then we just have to + // validate the pipeline Id. + return ContainerCommandRequestProto.newBuilder( + ContainerCommandRequestProto.parseFrom(request)) + .setPipelineID(gid.getUuid().toString()).build(); } private ContainerCommandResponseProto dispatchCommand( http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index f725d22..b46fd38 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -550,7 +550,8 @@ public class KeyValueContainer implements Container<KeyValueContainerData> { .setUsed(containerData.getBytesUsed()) .setState(getHddsState()) .setDeleteTransactionId(containerData.getDeleteTransactionId()) - .setBlockCommitSequenceId(containerData.getBlockCommitSequenceId()); + .setBlockCommitSequenceId(containerData.getBlockCommitSequenceId()) + .setOriginNodeId(containerData.getOriginNodeId()); return ciBuilder.build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java index f114d34..2a9eedc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java @@ -18,22 +18,17 @@ package org.apache.hadoop.ozone.container.keyvalue; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import java.util.Collections; -import org.apache.hadoop.conf.StorageSize; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerDataProto; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.yaml.snakeyaml.nodes.Tag; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -90,8 +85,10 @@ public class KeyValueContainerData extends ContainerData { * @param id - ContainerId * @param size - maximum size of the container in bytes */ - public KeyValueContainerData(long id, long size) { - super(ContainerProtos.ContainerType.KeyValueContainer, id, size); + public KeyValueContainerData(long id, long size, + String originPipelineId, String originNodeId) { + super(ContainerProtos.ContainerType.KeyValueContainer, id, size, + originPipelineId, originNodeId); this.numPendingDeletionBlocks = new AtomicInteger(0); this.deleteTransactionId = 0; } @@ -102,9 +99,10 @@ public class KeyValueContainerData extends ContainerData { * @param layOutVersion * @param size - maximum size of the container in bytes */ - public KeyValueContainerData(long id, int layOutVersion, long size) { + public KeyValueContainerData(long id, int layOutVersion, long size, + String originPipelineId, String originNodeId) { super(ContainerProtos.ContainerType.KeyValueContainer, id, layOutVersion, - size); + size, originPipelineId, originNodeId); this.numPendingDeletionBlocks = new AtomicInteger(0); this.deleteTransactionId = 0; } @@ -275,40 +273,4 @@ public class KeyValueContainerData extends ContainerData { return Collections.unmodifiableList(KV_YAML_FIELDS); } - /** - * Constructs a KeyValueContainerData object from ProtoBuf classes. - * - * @param protoData - ProtoBuf Message - * @throws IOException - */ - @VisibleForTesting - public static KeyValueContainerData getFromProtoBuf( - ContainerDataProto protoData) throws IOException { - // TODO: Add containerMaxSize to ContainerProtos.ContainerData - StorageSize storageSize = StorageSize.parse( - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); - KeyValueContainerData data = new KeyValueContainerData( - protoData.getContainerID(), - (long)storageSize.getUnit().toBytes(storageSize.getValue())); - for (int x = 0; x < protoData.getMetadataCount(); x++) { - data.addMetadata(protoData.getMetadata(x).getKey(), - protoData.getMetadata(x).getValue()); - } - - if (protoData.hasContainerPath()) { - String metadataPath = protoData.getContainerPath()+ File.separator + - OzoneConsts.CONTAINER_META_PATH; - data.setMetadataPath(metadataPath); - } - - if (protoData.hasState()) { - data.setState(protoData.getState()); - } - - if (protoData.hasBytesUsed()) { - data.setBytesUsed(protoData.getBytesUsed()); - } - - return data; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 4a5ef18..36c48c5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -220,7 +220,8 @@ public class KeyValueHandler extends Handler { long containerID = request.getContainerID(); KeyValueContainerData newContainerData = new KeyValueContainerData( - containerID, maxContainerSize); + containerID, maxContainerSize, request.getPipelineID(), + getDatanodeDetails().getUuidString()); // TODO: Add support to add metadataList to ContainerData. Add metadata // to container during creation. KeyValueContainer newContainer = new KeyValueContainer( @@ -772,13 +773,15 @@ public class KeyValueHandler extends Handler { } public Container importContainer(long containerID, long maxSize, + String originPipelineId, + String originNodeId, FileInputStream rawContainerStream, TarContainerPacker packer) throws IOException { KeyValueContainerData containerData = new KeyValueContainerData(containerID, - maxSize); + maxSize, originPipelineId, originNodeId); KeyValueContainer container = new KeyValueContainer(containerData, conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java index 41b74e7..1a918ed 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java @@ -105,11 +105,12 @@ public class ContainerController { } public Container importContainer(final ContainerType type, - final long containerId, final long maxSize, - final FileInputStream rawContainerStream, final TarContainerPacker packer) + final long containerId, final long maxSize, final String originPipelineId, + final String originNodeId, final FileInputStream rawContainerStream, + final TarContainerPacker packer) throws IOException { - return handlers.get(type).importContainer( - containerId, maxSize, rawContainerStream, packer); + return handlers.get(type).importContainer(containerId, maxSize, + originPipelineId, originNodeId, rawContainerStream, packer); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java index e8f0d17..3100a68 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java @@ -83,6 +83,8 @@ public class DownloadAndImportReplicator implements ContainerReplicator { originalContainerData.getContainerType(), containerID, originalContainerData.getMaxSize(), + originalContainerData.getOriginPipelineId(), + originalContainerData.getOriginNodeId(), tempContainerTarStream, packer); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto index 9fdef7d..c5f709c 100644 --- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -157,6 +157,7 @@ message ContainerReplicaProto { optional string finalhash = 10; optional int64 deleteTransactionId = 11; optional uint64 blockCommitSequenceId = 12; + optional string originNodeId = 13; } message CommandStatusReportsProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java index 824b770..c6fa8d6 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java @@ -25,6 +25,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; /** @@ -43,9 +44,11 @@ public class TestKeyValueContainerData { ContainerProtos.ContainerDataProto.State state = ContainerProtos.ContainerDataProto.State.CLOSED; AtomicLong val = new AtomicLong(0); + UUID pipelineId = UUID.randomUUID(); + UUID datanodeId = UUID.randomUUID(); KeyValueContainerData kvData = new KeyValueContainerData(containerId, - MAXSIZE); + MAXSIZE, pipelineId.toString(), datanodeId.toString()); assertEquals(containerType, kvData.getContainerType()); assertEquals(containerId, kvData.getContainerID()); @@ -83,6 +86,8 @@ public class TestKeyValueContainerData { assertEquals(1, kvData.getWriteCount()); assertEquals(1, kvData.getKeyCount()); assertEquals(1, kvData.getNumPendingDeletionBlocks()); + assertEquals(pipelineId.toString(), kvData.getOriginPipelineId()); + assertEquals(datanodeId.toString(), kvData.getOriginNodeId()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java index 087d627..a52e18b 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java @@ -29,6 +29,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -55,7 +56,8 @@ public class TestContainerDataYaml { String containerPath = containerID + ".container"; KeyValueContainerData keyValueContainerData = new KeyValueContainerData( - containerID, MAXSIZE); + containerID, MAXSIZE, UUID.randomUUID().toString(), + UUID.randomUUID().toString()); keyValueContainerData.setContainerDBType("RocksDB"); keyValueContainerData.setMetadataPath(testRoot); keyValueContainerData.setChunksPath(testRoot); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java index 7d8e438..d65d972 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -55,7 +56,8 @@ public class TestContainerSet { .ContainerDataProto.State.CLOSED; KeyValueContainerData kvData = new KeyValueContainerData(containerId, - (long) StorageUnit.GB.toBytes(5)); + (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(), + UUID.randomUUID().toString()); kvData.setState(state); KeyValueContainer keyValueContainer = new KeyValueContainer(kvData, new OzoneConfiguration()); @@ -166,7 +168,8 @@ public class TestContainerSet { ContainerSet containerSet = new ContainerSet(); for (int i=0; i<10; i++) { KeyValueContainerData kvData = new KeyValueContainerData(i, - (long) StorageUnit.GB.toBytes(5)); + (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(), + UUID.randomUUID().toString()); if (i%2 == 0) { kvData.setState(ContainerProtos.ContainerDataProto.State.CLOSED); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java index 085e6f9..9940d4d 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.protocol.proto import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Handler; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; @@ -76,9 +77,14 @@ public class TestHddsDispatcher { DatanodeDetails dd = randomDatanodeDetails(); ContainerSet containerSet = new ContainerSet(); VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf); + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd); + Mockito.when(context.getParent()).thenReturn(stateMachine); KeyValueContainerData containerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(1)); + (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), + dd.getUuidString()); Container container = new KeyValueContainer(containerData, conf); container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), scmId.toString()); @@ -87,8 +93,8 @@ public class TestHddsDispatcher { Map<ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerType containerType : ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType( - containerType, conf, null, containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, context, + containerSet, volumeSet, metrics)); } HddsDispatcher hddsDispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, context, metrics); @@ -125,13 +131,17 @@ public class TestHddsDispatcher { DatanodeDetails dd = randomDatanodeDetails(); ContainerSet containerSet = new ContainerSet(); VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf); + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd); + Mockito.when(context.getParent()).thenReturn(stateMachine); ContainerMetrics metrics = ContainerMetrics.create(conf); Map<ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerType containerType : ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType( - containerType, conf, null, containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, context, + containerSet, volumeSet, metrics)); } HddsDispatcher hddsDispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, context, metrics); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java index e763289..fb85194 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java @@ -20,10 +20,13 @@ package org.apache.hadoop.ozone.container.common.interfaces; import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; import org.junit.Assert; @@ -54,13 +57,19 @@ public class TestHandler { this.conf = new Configuration(); this.containerSet = Mockito.mock(ContainerSet.class); this.volumeSet = Mockito.mock(VolumeSet.class); + DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class); + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); + StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(stateMachine); ContainerMetrics metrics = ContainerMetrics.create(conf); Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { handlers.put(containerType, Handler.getHandlerForContainerType( - containerType, conf, null, containerSet, volumeSet, metrics)); + containerType, conf, context, containerSet, volumeSet, metrics)); } this.dispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, null, metrics); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java index 0cf95b7..a6fad88 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java @@ -51,7 +51,7 @@ import java.util.UUID; */ public class TestCloseContainerCommandHandler { - private static final StateContext CONTEXT = Mockito.mock(StateContext.class); + private final StateContext context = Mockito.mock(StateContext.class); private static File testDir; @@ -62,7 +62,12 @@ public class TestCloseContainerCommandHandler { conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath()); conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testDir.getPath()); - return new OzoneContainer(datanodeDetails, conf, CONTEXT); + final DatanodeStateMachine datanodeStateMachine = Mockito.mock( + DatanodeStateMachine.class); + Mockito.when(datanodeStateMachine.getDatanodeDetails()) + .thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(datanodeStateMachine); + return new OzoneContainer(datanodeDetails, conf, context); } @@ -106,20 +111,14 @@ public class TestCloseContainerCommandHandler { new CloseContainerCommandHandler(); final CloseContainerCommand command = new CloseContainerCommand( containerId.getId(), pipelineID); - final DatanodeStateMachine datanodeStateMachine = Mockito.mock( - DatanodeStateMachine.class); - - Mockito.when(datanodeStateMachine.getDatanodeDetails()) - .thenReturn(datanodeDetails); - Mockito.when(CONTEXT.getParent()).thenReturn(datanodeStateMachine); - closeHandler.handle(command, container, CONTEXT, null); + closeHandler.handle(command, container, context, null); Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, container.getContainerSet().getContainer( containerId.getId()).getContainerState()); - Mockito.verify(datanodeStateMachine, Mockito.times(2)).triggerHeartbeat(); + Mockito.verify(context.getParent(), Mockito.times(2)).triggerHeartbeat(); container.stop(); } @@ -164,20 +163,14 @@ public class TestCloseContainerCommandHandler { // Specify a pipeline which doesn't exist in the datanode. final CloseContainerCommand command = new CloseContainerCommand( containerId.getId(), PipelineID.randomId()); - final DatanodeStateMachine datanodeStateMachine = Mockito.mock( - DatanodeStateMachine.class); - - Mockito.when(datanodeStateMachine.getDatanodeDetails()) - .thenReturn(datanodeDetails); - Mockito.when(CONTEXT.getParent()).thenReturn(datanodeStateMachine); - closeHandler.handle(command, container, CONTEXT, null); + closeHandler.handle(command, container, context, null); Assert.assertEquals(ContainerProtos.ContainerDataProto.State.QUASI_CLOSED, container.getContainerSet().getContainer( containerId.getId()).getContainerState()); - Mockito.verify(datanodeStateMachine, Mockito.times(2)).triggerHeartbeat(); + Mockito.verify(context.getParent(), Mockito.times(2)).triggerHeartbeat(); container.stop(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java index e3e683c..1d580a0 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java @@ -68,9 +68,9 @@ public class TestBlockManagerImpl { @Before public void setUp() throws Exception { config = new OzoneConfiguration(); - + UUID datanodeId = UUID.randomUUID(); HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID() + .getAbsolutePath()).conf(config).datanodeUuid(datanodeId .toString()).build(); volumeSet = mock(VolumeSet.class); @@ -80,7 +80,8 @@ public class TestBlockManagerImpl { .thenReturn(hddsVolume); keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); + (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(), + datanodeId.toString()); keyValueContainer = new KeyValueContainer( keyValueContainerData, config); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java index 3f181d1..abb9059 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java @@ -72,8 +72,9 @@ public class TestChunkManagerImpl { @Before public void setUp() throws Exception { config = new OzoneConfiguration(); + UUID datanodeId = UUID.randomUUID(); hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID() + .getAbsolutePath()).conf(config).datanodeUuid(datanodeId .toString()).build(); volumeSet = mock(VolumeSet.class); @@ -83,7 +84,8 @@ public class TestChunkManagerImpl { .thenReturn(hddsVolume); keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); + (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(), + datanodeId.toString()); keyValueContainer = new KeyValueContainer(keyValueContainerData, config); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java index 5fa7b54..15d7b34 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java @@ -246,7 +246,8 @@ public class TestKeyValueBlockIterator { normalBlocks, int deletedBlocks) throws Exception { containerData = new KeyValueContainerData(containerId, - (long) StorageUnit.GB.toBytes(1)); + (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), + UUID.randomUUID().toString()); container = new KeyValueContainer(containerData, conf); container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID .randomUUID().toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index 770a9f7..002d375 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -81,12 +81,14 @@ public class TestKeyValueContainer { private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; private KeyValueContainerData keyValueContainerData; private KeyValueContainer keyValueContainer; + private UUID datanodeId; @Before public void setUp() throws Exception { conf = new OzoneConfiguration(); + datanodeId = UUID.randomUUID(); HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(conf).datanodeUuid(UUID.randomUUID() + .getAbsolutePath()).conf(conf).datanodeUuid(datanodeId .toString()).build(); volumeSet = mock(VolumeSet.class); @@ -95,7 +97,8 @@ public class TestKeyValueContainer { .thenReturn(hddsVolume); keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); + (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(), + datanodeId.toString()); keyValueContainer = new KeyValueContainer( keyValueContainerData, conf); @@ -105,7 +108,8 @@ public class TestKeyValueContainer { @Test public void testBlockIterator() throws Exception{ keyValueContainerData = new KeyValueContainerData(100L, - (long) StorageUnit.GB.toBytes(1)); + (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), + datanodeId.toString()); keyValueContainer = new KeyValueContainer( keyValueContainerData, conf); keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); @@ -214,7 +218,8 @@ public class TestKeyValueContainer { //create a new one KeyValueContainerData containerData = new KeyValueContainerData(containerId, 1, - keyValueContainerData.getMaxSize()); + keyValueContainerData.getMaxSize(), UUID.randomUUID().toString(), + datanodeId.toString()); KeyValueContainer container = new KeyValueContainer(containerData, conf); HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index 265fb18..698af35 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; @@ -32,6 +33,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -225,7 +228,14 @@ public class TestKeyValueHandler { interval[0] = 2; ContainerMetrics metrics = new ContainerMetrics(interval); VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); - KeyValueHandler keyValueHandler = new KeyValueHandler(conf, null, cset, + DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class); + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); + StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()) + .thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(stateMachine); + KeyValueHandler keyValueHandler = new KeyValueHandler(conf, context, cset, volumeSet, metrics); assertEquals("org.apache.hadoop.ozone.container.common" + ".volume.RoundRobinVolumeChoosingPolicy", @@ -236,7 +246,7 @@ public class TestKeyValueHandler { conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY, "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher"); try { - new KeyValueHandler(conf, null, cset, volumeSet, metrics); + new KeyValueHandler(conf, context, cset, volumeSet, metrics); } catch (RuntimeException ex) { GenericTestUtils.assertExceptionContains("class org.apache.hadoop" + ".ozone.container.common.impl.HddsDispatcher not org.apache" + @@ -266,7 +276,8 @@ public class TestKeyValueHandler { long containerID = 1234L; Configuration conf = new Configuration(); KeyValueContainerData kvData = new KeyValueContainerData(containerID, - (long) StorageUnit.GB.toBytes(1)); + (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), + UUID.randomUUID().toString()); KeyValueContainer container = new KeyValueContainer(kvData, conf); kvData.setState(ContainerProtos.ContainerDataProto.State.INVALID); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java index 1a92ca4..9e6f653 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java @@ -29,6 +29,7 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker; @@ -91,7 +92,8 @@ public class TestTarContainerPacker { Files.createDirectories(dbDir); Files.createDirectories(dataDir); - KeyValueContainerData containerData = new KeyValueContainerData(id, -1); + KeyValueContainerData containerData = new KeyValueContainerData( + id, -1, UUID.randomUUID().toString(), UUID.randomUUID().toString()); containerData.setChunksPath(dataDir.toString()); containerData.setMetadataPath(dbDir.getParent().toString()); containerData.setDbFile(dbDir.toFile()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 2326818..388d0c1 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; @@ -34,6 +36,8 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + import java.util.Random; import java.util.UUID; @@ -79,16 +83,22 @@ public class TestOzoneContainer { // Add containers to disk for (int i=0; i<10; i++) { keyValueContainerData = new KeyValueContainerData(i, - (long) StorageUnit.GB.toBytes(1)); + (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), + datanodeDetails.getUuidString()); keyValueContainer = new KeyValueContainer( keyValueContainerData, conf); keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); } + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); + StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(stateMachine); // When OzoneContainer is started, the containers from disk should be // loaded into the containerSet. OzoneContainer ozoneContainer = new - OzoneContainer(datanodeDetails, conf, null); + OzoneContainer(datanodeDetails, conf, context); ContainerSet containerset = ozoneContainer.getContainerSet(); assertEquals(10, containerset.containerCount()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java index 030412c..c3d3b17 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.replication; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -118,7 +119,8 @@ public class TestReplicationSupervisor { @Override public void replicate(ReplicationTask task) { KeyValueContainerData kvcd = - new KeyValueContainerData(task.getContainerId(), 100L); + new KeyValueContainerData(task.getContainerId(), 100L, + UUID.randomUUID().toString(), UUID.randomUUID().toString()); KeyValueContainer kvc = new KeyValueContainer(kvcd, conf); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/resources/additionalfields.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/additionalfields.container b/hadoop-hdds/container-service/src/test/resources/additionalfields.container index fff5304..faaed06 100644 --- a/hadoop-hdds/container-service/src/test/resources/additionalfields.container +++ b/hadoop-hdds/container-service/src/test/resources/additionalfields.container @@ -6,7 +6,9 @@ containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1 layOutVersion: 1 maxSize: 5368709120 +originPipelineId: 1297e8a9-2850-4ced-b96c-5ae31d2c73ad +originNodeId: 7f541a06-6c26-476d-9994-c6e1947e11cb metadata: {OWNER: ozone, VOLUME: hdfs} state: CLOSED aclEnabled: true -checksum: c5b5373b8755c4e7199478dcaded9d996f9aca089704e08950259cdb0f290680 \ No newline at end of file +checksum: 61db56da7d50798561b5365c123c5fbf7faf99fbbbd571a746af79020b7f79ba \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container index d06ba57..ce32947 100644 --- a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container +++ b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container @@ -6,6 +6,8 @@ containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1 layOutVersion: 1 maxSize: 5368709120 +originPipelineId: 4d41dd20-6d73-496a-b247-4c6cb483f54e +originNodeId: 54842560-67a5-48a5-a7d4-4701d9538706 metadata: {OWNER: ozone, VOLUME: hdfs} state: OPEN checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/resources/incorrect.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.container b/hadoop-hdds/container-service/src/test/resources/incorrect.container index 0053ab2..d14c75f 100644 --- a/hadoop-hdds/container-service/src/test/resources/incorrect.container +++ b/hadoop-hdds/container-service/src/test/resources/incorrect.container @@ -6,6 +6,8 @@ containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1 layOutVersion: 1 maxSize: 5368709120 +originPipelineId: b2c96aa4-b757-4f97-b286-6fb80a1baf8e +originNodeId: 6dcfb385-caea-4efb-9ef3-f87fadca0f51 metadata: {OWNER: ozone, VOLUME: hdfs} state: INVALID checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index 16c9f22..600ee5a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -154,8 +154,9 @@ public class TestEndPoint { OzoneConfiguration conf = SCMTestUtils.getConf(); try (EndpointStateMachine rpcEndPoint = createEndpoint(conf, serverAddress, 1000)) { + DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer( - TestUtils.randomDatanodeDetails(), conf, null); + datanodeDetails, conf, getContext(datanodeDetails)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, conf, ozoneContainer); @@ -178,8 +179,9 @@ public class TestEndPoint { serverAddress, 1000)) { GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer .captureLogs(VersionEndpointTask.LOG); - OzoneContainer ozoneContainer = new OzoneContainer(TestUtils - .randomDatanodeDetails(), conf, null); + DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); + OzoneContainer ozoneContainer = new OzoneContainer( + datanodeDetails, conf, getContext(datanodeDetails)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, conf, ozoneContainer); @@ -231,8 +233,9 @@ public class TestEndPoint { try (EndpointStateMachine rpcEndPoint = createEndpoint(conf, nonExistentServerAddress, 1000)) { rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); + DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer( - TestUtils.randomDatanodeDetails(), conf, null); + datanodeDetails, conf, getContext(datanodeDetails)); VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, conf, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); @@ -258,8 +261,9 @@ public class TestEndPoint { try (EndpointStateMachine rpcEndPoint = createEndpoint(conf, serverAddress, (int) rpcTimeout)) { rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); + DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer( - TestUtils.randomDatanodeDetails(), conf, null); + datanodeDetails, conf, getContext(datanodeDetails)); VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, conf, ozoneContainer); @@ -527,4 +531,13 @@ public class TestEndPoint { lessThanOrEqualTo(rpcTimeout + tolerance)); } + private StateContext getContext(DatanodeDetails datanodeDetails) { + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); + StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(stateMachine); + return context; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index 7e30c5f..c2fb2ea 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -110,7 +110,8 @@ public class TestBlockDeletingService { conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath()); long containerID = ContainerTestHelper.getTestContainerID(); KeyValueContainerData data = new KeyValueContainerData(containerID, - ContainerTestHelper.CONTAINER_MAX_SIZE); + ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(), + UUID.randomUUID().toString()); Container container = new KeyValueContainer(data, conf); container.create(new VolumeSet(scmId, clusterID, conf), new RoundRobinVolumeChoosingPolicy(), scmId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java index 6478fc6..745f730 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java @@ -24,6 +24,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomUtils; @@ -72,7 +73,8 @@ public class TestContainerDeletionChoosingPolicy { int numContainers = 10; for (int i = 0; i < numContainers; i++) { KeyValueContainerData data = new KeyValueContainerData(i, - ContainerTestHelper.CONTAINER_MAX_SIZE); + ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(), + UUID.randomUUID().toString()); KeyValueContainer container = new KeyValueContainer(data, conf); containerSet.addContainer(container); Assert.assertTrue( @@ -125,7 +127,9 @@ public class TestContainerDeletionChoosingPolicy { long containerId = RandomUtils.nextLong(); KeyValueContainerData data = new KeyValueContainerData(containerId, - ContainerTestHelper.CONTAINER_MAX_SIZE); + ContainerTestHelper.CONTAINER_MAX_SIZE, + UUID.randomUUID().toString(), + UUID.randomUUID().toString()); if (i != numContainers) { int deletionBlocks = random.nextInt(numContainers) + 1; data.incrPendingDeletionBlocks(deletionBlocks); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index f6c498a..a48d637 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -157,7 +157,8 @@ public class TestContainerPersistence { private Container addContainer(ContainerSet cSet, long cID) throws IOException { KeyValueContainerData data = new KeyValueContainerData(cID, - ContainerTestHelper.CONTAINER_MAX_SIZE); + ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(), + UUID.randomUUID().toString()); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); KeyValueContainer container = new KeyValueContainer(data, conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 527ab45..44c1172 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -41,6 +41,8 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Handler; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.hdds.scm.TestUtils; @@ -51,6 +53,7 @@ import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicatio import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.File; import java.util.Map; @@ -90,16 +93,22 @@ public class TestContainerMetrics { VolumeSet volumeSet = new VolumeSet( datanodeDetails.getUuidString(), conf); ContainerSet containerSet = new ContainerSet(); + DatanodeStateMachine stateMachine = Mockito.mock( + DatanodeStateMachine.class); + StateContext context = Mockito.mock(StateContext.class); + Mockito.when(stateMachine.getDatanodeDetails()) + .thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(stateMachine); ContainerMetrics metrics = ContainerMetrics.create(conf); Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType( - containerType, conf, null, containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, context, + containerSet, volumeSet, metrics)); } HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet, - volumeSet, handlers, null, metrics); + volumeSet, handlers, context, metrics); dispatcher.setScmId(UUID.randomUUID().toString()); server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 3a15b21..cd0b060 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -29,11 +29,14 @@ import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.XceiverClientGrpc; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.rules.Timeout; +import org.mockito.Mockito; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -72,8 +75,13 @@ public class TestOzoneContainer { conf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); + DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class); + StateContext context = Mockito.mock(StateContext.class); + DatanodeStateMachine dsm = Mockito.mock(DatanodeStateMachine.class); + Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails); + Mockito.when(context.getParent()).thenReturn(dsm); container = new OzoneContainer(TestUtils.randomDatanodeDetails(), - conf, null); + conf, context); //Setting scmId, as we start manually ozone container. container.getDispatcher().setScmId(UUID.randomUUID().toString()); container.start(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
