This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 04c62c5 HDDS-6411. EC: OmMultipartKeyInfo needs to handle
ECReplicationConfig (#3210)
04c62c5 is described below
commit 04c62c53f280083996403117eb360c6363bb5e57
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Mar 22 19:30:23 2022 +0100
HDDS-6411. EC: OmMultipartKeyInfo needs to handle ECReplicationConfig
(#3210)
---
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +++-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 111 +++++++++++++++++----
.../src/main/proto/OmClientProtocol.proto | 3 +-
3 files changed, 107 insertions(+), 26 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
index 41779ab..86da3eb 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.om.helpers;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartKeyInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
@@ -200,10 +201,11 @@ public class OmMultipartKeyInfo extends WithObjectID {
multipartKeyInfo.getPartKeyInfoListList().forEach(partKeyInfo ->
list.put(partKeyInfo.getPartNumber(), partKeyInfo));
- final ReplicationConfig replicationConfig = ReplicationConfig
- .fromProtoTypeAndFactor(
- multipartKeyInfo.getType(),
- multipartKeyInfo.getFactor());
+ final ReplicationConfig replicationConfig = ReplicationConfig.fromProto(
+ multipartKeyInfo.getType(),
+ multipartKeyInfo.getFactor(),
+ multipartKeyInfo.getEcReplicationConfig()
+ );
return new OmMultipartKeyInfo(multipartKeyInfo.getUploadID(),
multipartKeyInfo.getCreationTime(), replicationConfig,
@@ -220,10 +222,17 @@ public class OmMultipartKeyInfo extends WithObjectID {
.setUploadID(uploadID)
.setCreationTime(creationTime)
.setType(replicationConfig.getReplicationType())
- .setFactor(ReplicationConfig.getLegacyFactor(replicationConfig))
.setObjectID(objectID)
.setUpdateID(updateID)
.setParentID(parentID);
+
+ if (replicationConfig instanceof ECReplicationConfig) {
+ ECReplicationConfig ecConf = (ECReplicationConfig) replicationConfig;
+ builder.setEcReplicationConfig(ecConf.toProto());
+ } else {
+ builder.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig));
+ }
+
partKeyInfoList.forEach((key, value) -> builder.addPartKeyInfoList(value));
return builder.build();
}
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
index b498f83..59aeece 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
@@ -18,15 +18,23 @@
package org.apache.hadoop.ozone.om.helpers;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
import org.apache.hadoop.util.Time;
-import org.junit.Assert;
import org.junit.Test;
import java.util.UUID;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
/**
* Class to test OmMultipartKeyInfo.
@@ -34,33 +42,96 @@ import java.util.UUID;
public class TestOmMultipartKeyInfo {
@Test
- public void testCopyObject() {
- OmMultipartKeyInfo omMultipartKeyInfo = new OmMultipartKeyInfo.Builder()
- .setUploadID(UUID.randomUUID().toString())
- .setCreationTime(Time.now())
- .setReplicationConfig(
- new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE))
+ public void copyObject() {
+ for (ReplicationConfig param : replicationConfigs().collect(toList())) {
+ testCopyObject(param);
+ }
+ }
+
+ //@ParameterizedTest
+ //@MethodSource("replicationConfigs")
+ private void testCopyObject(ReplicationConfig replicationConfig) {
+ // GIVEN
+ OmMultipartKeyInfo subject = createSubject()
+ .setReplicationConfig(replicationConfig)
+ .build();
+
+ // WHEN
+ OmMultipartKeyInfo copy = subject.copyObject();
+
+ // THEN
+ assertNotSame(subject, copy);
+ assertEquals(subject, copy);
+ assertEquals(replicationConfig, copy.getReplicationConfig());
+ }
+
+ @Test
+ public void protoConversion() {
+ for (ReplicationConfig param : replicationConfigs().collect(toList())) {
+ protoConversion(param);
+ }
+ }
+
+ //@ParameterizedTest
+ //@MethodSource("replicationConfigs")
+ private void protoConversion(ReplicationConfig replicationConfig) {
+ // GIVEN
+ OmMultipartKeyInfo subject = createSubject()
+ .setReplicationConfig(replicationConfig)
.build();
- OmMultipartKeyInfo cloneMultipartKeyInfo = omMultipartKeyInfo.copyObject();
+ // WHEN
+ OzoneManagerProtocolProtos.MultipartKeyInfo proto = subject.getProto();
+ OmMultipartKeyInfo fromProto = OmMultipartKeyInfo.getFromProto(proto);
+
+ // THEN
+ assertEquals(subject, fromProto);
+ assertEquals(replicationConfig, fromProto.getReplicationConfig());
+ }
+
+ private static Stream<ReplicationConfig> replicationConfigs() {
+ return Stream.of(
+ new StandaloneReplicationConfig(HddsProtos.ReplicationFactor.ONE),
+ new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE),
+ new ECReplicationConfig(3, 2)
+ );
+ }
+
+ @Test
+ public void distinctListOfParts() {
+ // GIVEN
+ OmMultipartKeyInfo subject = createSubject().build();
+ OmMultipartKeyInfo copy = subject.copyObject();
+
+ // WHEN
+ subject.addPartKeyInfo(1, createPart(createKeyInfo()).build());
+
+ // THEN
+ assertEquals(0, copy.getPartKeyInfoMap().size());
+ assertEquals(1, subject.getPartKeyInfoMap().size());
+ }
- Assert.assertEquals(cloneMultipartKeyInfo, omMultipartKeyInfo);
+ private static OmMultipartKeyInfo.Builder createSubject() {
+ return new OmMultipartKeyInfo.Builder()
+ .setUploadID(UUID.randomUUID().toString())
+ .setCreationTime(Time.now());
+ }
- // Just setting dummy values for this test.
- omMultipartKeyInfo.addPartKeyInfo(1,
- PartKeyInfo.newBuilder().setPartNumber(1).setPartName("/path")
- .setPartKeyInfo(KeyInfo.newBuilder()
+ private static PartKeyInfo.Builder createPart(KeyInfo.Builder partKeyInfo) {
+ return PartKeyInfo.newBuilder()
+ .setPartNumber(1)
+ .setPartName("/path")
+ .setPartKeyInfo(partKeyInfo);
+ }
+
+ private static KeyInfo.Builder createKeyInfo() {
+ return KeyInfo.newBuilder()
.setVolumeName(UUID.randomUUID().toString())
.setBucketName(UUID.randomUUID().toString())
.setKeyName(UUID.randomUUID().toString())
- .setDataSize(100L) // Just set dummy size for testing
+ .setDataSize(100L)
.setCreationTime(Time.now())
.setModificationTime(Time.now())
- .setType(HddsProtos.ReplicationType.RATIS)
- .setFactor(HddsProtos.ReplicationFactor.ONE).build()).build());
-
- Assert.assertEquals(0, cloneMultipartKeyInfo.getPartKeyInfoMap().size());
- Assert.assertEquals(1, omMultipartKeyInfo.getPartKeyInfoMap().size());
-
+ .setType(HddsProtos.ReplicationType.STAND_ALONE);
}
}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index fe8b0d1..1e89f5e 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1214,11 +1214,12 @@ message MultipartKeyInfo {
required string uploadID = 1;
required uint64 creationTime = 2;
required hadoop.hdds.ReplicationType type = 3;
- required hadoop.hdds.ReplicationFactor factor = 4;
+ optional hadoop.hdds.ReplicationFactor factor = 4;
repeated PartKeyInfo partKeyInfoList = 5;
optional uint64 objectID = 6;
optional uint64 updateID = 7;
optional uint64 parentID = 8;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 9;
}
message PartKeyInfo {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
