Repository: hadoop Updated Branches: refs/heads/trunk 0ce6290de -> 41ae5c500
HDDS-77. Key replication factor and type should be stored per key by Ozone Manager. Contributed by Mukul Kumar Singh. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41ae5c50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41ae5c50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41ae5c50 Branch: refs/heads/trunk Commit: 41ae5c50024e3682b61df6f391d0fe63591ac92b Parents: 0ce6290 Author: Xiaoyu Yao <x...@apache.org> Authored: Thu May 17 08:31:44 2018 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Thu May 17 08:31:44 2018 -0700 ---------------------------------------------------------------------- .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 36 ++++++++++++++++++-- ...ceManagerProtocolClientSideTranslatorPB.java | 2 -- .../main/proto/KeySpaceManagerProtocol.proto | 10 +++--- .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 19 +++-------- ...ceManagerProtocolServerSideTranslatorPB.java | 6 ---- 5 files changed, 44 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java index 678ce92..5d6e633 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.ksm.helpers; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo; import org.apache.hadoop.util.Time; @@ -39,10 +40,13 @@ public final class KsmKeyInfo { private List<KsmKeyLocationInfoGroup> keyLocationVersions; private final long creationTime; private long modificationTime; + private HddsProtos.ReplicationType type; + private HddsProtos.ReplicationFactor factor; private KsmKeyInfo(String volumeName, String bucketName, String keyName, List<KsmKeyLocationInfoGroup> versions, long dataSize, - long creationTime, long modificationTime) { + long creationTime, long modificationTime, HddsProtos.ReplicationType type, + HddsProtos.ReplicationFactor factor) { this.volumeName = volumeName; this.bucketName = bucketName; this.keyName = keyName; @@ -61,6 +65,8 @@ public final class KsmKeyInfo { this.keyLocationVersions = versions; this.creationTime = creationTime; this.modificationTime = modificationTime; + this.factor = factor; + this.type = type; } public String getVolumeName() { @@ -71,6 +77,14 @@ public final class KsmKeyInfo { return bucketName; } + public HddsProtos.ReplicationType getType() { + return type; + } + + public HddsProtos.ReplicationFactor getFactor() { + return factor; + } + public String getKeyName() { return keyName; } @@ -170,6 +184,8 @@ public final class KsmKeyInfo { private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups; private long creationTime; private long modificationTime; + private HddsProtos.ReplicationType type; + private HddsProtos.ReplicationFactor factor; public Builder setVolumeName(String volume) { this.volumeName = volume; @@ -207,10 +223,20 @@ public final class KsmKeyInfo { return this; } + public Builder setReplicationFactor(HddsProtos.ReplicationFactor factor) { + this.factor = factor; + return this; + } + + public Builder setReplicationType(HddsProtos.ReplicationType type) { + this.type = type; + return this; + } + public KsmKeyInfo build() { return new KsmKeyInfo( volumeName, bucketName, keyName, ksmKeyLocationInfoGroups, - dataSize, creationTime, modificationTime); + dataSize, creationTime, modificationTime, type, factor); } } @@ -222,6 +248,8 @@ public final class KsmKeyInfo { .setBucketName(bucketName) .setKeyName(keyName) .setDataSize(dataSize) + .setFactor(factor) + .setType(type) .addAllKeyLocationList(keyLocationVersions.stream() .map(KsmKeyLocationInfoGroup::getProtobuf) .collect(Collectors.toList())) @@ -241,7 +269,9 @@ public final class KsmKeyInfo { .collect(Collectors.toList()), keyInfo.getDataSize(), keyInfo.getCreationTime(), - keyInfo.getModificationTime()); + keyInfo.getModificationTime(), + keyInfo.getType(), + keyInfo.getFactor()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java index 854c688..0f38169 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java @@ -560,8 +560,6 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB .setVolumeName(args.getVolumeName()) .setBucketName(args.getBucketName()) .setKeyName(args.getKeyName()) - .setFactor(args.getFactor()) - .setType(args.getType()) .setDataSize(args.getDataSize()).build(); req.setKeyArgs(keyArgs); req.setClientID(clientID); http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto index 7b70330..d3d1de6 100644 --- a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto @@ -249,10 +249,12 @@ message KeyInfo { required string bucketName = 2; required string keyName = 3; required uint64 dataSize = 4; - repeated KeyLocationList keyLocationList = 5; - required uint64 creationTime = 6; - required uint64 modificationTime = 7; - optional uint64 latestVersion = 8; + required hadoop.hdds.ReplicationType type = 5; + required hadoop.hdds.ReplicationFactor factor = 6; + repeated KeyLocationList keyLocationList = 7; + required uint64 creationTime = 8; + required uint64 modificationTime = 9; + optional uint64 latestVersion = 10; } message LocateKeyRequest { http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java index 6409a73..0d4cfda 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java @@ -173,18 +173,6 @@ public class KeyManagerImpl implements KeyManager { String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); String keyName = args.getKeyName(); - ReplicationFactor factor = args.getFactor(); - ReplicationType type = args.getType(); - - // If user does not specify a replication strategy or - // replication factor, KSM will use defaults. - if(factor == null) { - factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE; - } - - if(type == null) { - type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE; - } try { validateBucket(volumeName, bucketName); @@ -198,10 +186,11 @@ public class KeyManagerImpl implements KeyManager { throw new KSMException("Open Key not found", KSMException.ResultCodes.FAILED_KEY_NOT_FOUND); } - AllocatedBlock allocatedBlock = - scmBlockClient.allocateBlock(scmBlockSize, type, factor, ksmId); KsmKeyInfo keyInfo = KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData)); + AllocatedBlock allocatedBlock = + scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(), + keyInfo.getFactor(), ksmId); KsmKeyLocationInfo info = new KsmKeyLocationInfo.Builder() .setBlockID(allocatedBlock.getBlockID()) .setShouldCreateContainer(allocatedBlock.getCreateContainer()) @@ -293,6 +282,8 @@ public class KeyManagerImpl implements KeyManager { .setCreationTime(currentTime) .setModificationTime(currentTime) .setDataSize(size) + .setReplicationType(type) + .setReplicationFactor(factor) .build(); openVersion = 0; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java index 536f95a..38e7797 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java @@ -527,16 +527,10 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements AllocateBlockResponse.newBuilder(); try { KeyArgs keyArgs = request.getKeyArgs(); - HddsProtos.ReplicationType type = - keyArgs.hasType()? keyArgs.getType() : null; - HddsProtos.ReplicationFactor factor = - keyArgs.hasFactor()? keyArgs.getFactor() : null; KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder() .setVolumeName(keyArgs.getVolumeName()) .setBucketName(keyArgs.getBucketName()) .setKeyName(keyArgs.getKeyName()) - .setType(type) - .setFactor(factor) .build(); int id = request.getClientID(); KsmKeyLocationInfo newLocation = impl.allocateBlock(ksmKeyArgs, id); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
