This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 460aa9b HDDS-6020: EC: Provide set replicationConfig option to bucket
(#2880)
460aa9b is described below
commit 460aa9b89bb8a27eef4e17a9864f86da33763bfb
Author: Uma Maheswara Rao G <[email protected]>
AuthorDate: Tue Dec 7 16:59:56 2021 -0800
HDDS-6020: EC: Provide set replicationConfig option to bucket (#2880)
Co-authored-by: Uma Maheswara Rao G <[email protected]>
---
.../hdds/client/DefaultReplicationConfig.java | 24 +++++++++
.../apache/hadoop/ozone/client/OzoneBucket.java | 11 +++++
.../ozone/client/protocol/ClientProtocol.java | 11 +++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 ++++++
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 57 +++++++++++++++++-----
.../client/rpc/TestOzoneRpcClientAbstract.java | 47 ++++++++++++++++++
.../request/bucket/OMBucketSetPropertyRequest.java | 8 +++
7 files changed, 162 insertions(+), 11 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
index a11b7fc..dda778f 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
@@ -37,6 +37,17 @@ public class DefaultReplicationConfig {
this.ecReplicationConfig = null;
}
+ public DefaultReplicationConfig(ReplicationConfig replicationConfig) {
+ this.type =
+ ReplicationType.fromProto(replicationConfig.getReplicationType());
+ if (replicationConfig instanceof ECReplicationConfig) {
+ this.ecReplicationConfig = (ECReplicationConfig) replicationConfig;
+ } else {
+ this.factor =
+ ReplicationFactor.valueOf(replicationConfig.getRequiredNodes());
+ }
+ }
+
public DefaultReplicationConfig(ReplicationType type,
ECReplicationConfig ecReplicationConfig) {
this.type = type;
@@ -51,6 +62,19 @@ public class DefaultReplicationConfig {
this.ecReplicationConfig = ecReplicationConfig;
}
+ public DefaultReplicationConfig(
+ org.apache.hadoop.hdds.protocol.proto.HddsProtos.DefaultReplicationConfig
+ defaultReplicationConfig) {
+ this.type = ReplicationType.fromProto(defaultReplicationConfig.getType());
+ if (defaultReplicationConfig.hasEcReplicationConfig()) {
+ this.ecReplicationConfig = new ECReplicationConfig(
+ defaultReplicationConfig.getEcReplicationConfig());
+ } else {
+ this.factor =
+ ReplicationFactor.fromProto(defaultReplicationConfig.getFactor());
+ }
+ }
+
public ReplicationType getType() {
return this.type;
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 575612d..37098a9 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -514,6 +514,17 @@ public class OzoneBucket extends WithMetadata {
}
/**
+ * Sets/Changes the default replication config of this Bucket.
+ *
+ * @param replicationConfig Replication config that can be applied to bucket.
+ * @throws IOException
+ */
+ public void setReplicationConfig(ReplicationConfig replicationConfig)
+ throws IOException {
+ proxy.setReplicationConfig(volumeName, name, replicationConfig);
+ }
+
+ /**
* Creates a new key in the bucket, with default replication type RATIS and
* with replication factor THREE.
* @param key Name of the key to be created.
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 2feb577..c2280ed 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -739,6 +739,17 @@ public interface ClientProtocol {
long quotaInNamespace, long quotaInBytes) throws IOException;
/**
+ * Set Bucket replication configuration.
+ *
+ * @param volumeName Name of the Volume.
+ * @param bucketName Name of the Bucket.
+ * @param replicationConfig The replication config to set on bucket.
+ * @throws IOException
+ */
+ void setReplicationConfig(String volumeName, String bucketName,
+ ReplicationConfig replicationConfig) throws IOException;
+
+ /**
* Returns OzoneKey that contains the application generated/visible
* metadata for an Ozone Object.
*
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index da0b4c1..7fdb8fc 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -705,6 +705,21 @@ public class RpcClient implements ClientProtocol {
}
@Override
+ public void setReplicationConfig(
+ String volumeName, String bucketName, ReplicationConfig
replicationConfig)
+ throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ Preconditions.checkNotNull(replicationConfig);
+ OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
+ builder.setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setDefaultReplicationConfig(
+ new DefaultReplicationConfig(replicationConfig));
+ ozoneManagerClient.setBucketProperty(builder.build());
+ }
+
+ @Override
public void deleteBucket(
String volumeName, String bucketName) throws IOException {
verifyVolumeName(volumeName);
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
index 1c8c18a..56e825f 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.om.helpers;
import java.util.LinkedHashMap;
import java.util.Map;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.Auditable;
@@ -51,6 +52,7 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
private long quotaInBytes;
private long quotaInNamespace;
+ private DefaultReplicationConfig defaultReplicationConfig = null;
/**
* Private constructor, constructed via builder.
@@ -122,6 +124,22 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
}
/**
+ * Returns Bucket default replication config.
+ * @return
+ */
+ public DefaultReplicationConfig getDefaultReplicationConfig() {
+ return defaultReplicationConfig;
+ }
+
+ /**
+ * Sets the Bucket default replication config.
+ */
+ private void setDefaultReplicationConfig(
+ DefaultReplicationConfig defaultReplicationConfig) {
+ this.defaultReplicationConfig = defaultReplicationConfig;
+ }
+
+ /**
* Returns new builder class that builds a OmBucketArgs.
* @return Builder
*/
@@ -155,6 +173,7 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
private Map<String, String> metadata;
private long quotaInBytes;
private long quotaInNamespace;
+ private DefaultReplicationConfig defaultReplicationConfig;
/**
* Constructs a builder.
@@ -199,6 +218,12 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
return this;
}
+ public Builder setDefaultReplicationConfig(
+ DefaultReplicationConfig defaultRepConfig) {
+ this.defaultReplicationConfig = defaultRepConfig;
+ return this;
+ }
+
/**
* Constructs the OmBucketArgs.
* @return instance of OmBucketArgs.
@@ -206,8 +231,11 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
public OmBucketArgs build() {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
- return new OmBucketArgs(volumeName, bucketName, isVersionEnabled,
- storageType, metadata, quotaInBytes, quotaInNamespace);
+ OmBucketArgs omBucketArgs =
+ new OmBucketArgs(volumeName, bucketName, isVersionEnabled,
+ storageType, metadata, quotaInBytes, quotaInNamespace);
+ omBucketArgs.setDefaultReplicationConfig(defaultReplicationConfig);
+ return omBucketArgs;
}
}
@@ -230,6 +258,9 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
if(quotaInNamespace > 0 || quotaInNamespace == OzoneConsts.QUOTA_RESET) {
builder.setQuotaInNamespace(quotaInNamespace);
}
+ if (defaultReplicationConfig != null) {
+ builder.setDefaultReplicationConfig(defaultReplicationConfig.toProto());
+ }
return builder.build();
}
@@ -239,14 +270,18 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
* @return instance of OmBucketArgs
*/
public static OmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
- return new OmBucketArgs(bucketArgs.getVolumeName(),
- bucketArgs.getBucketName(),
- bucketArgs.hasIsVersionEnabled() ?
- bucketArgs.getIsVersionEnabled() : null,
- bucketArgs.hasStorageType() ? StorageType.valueOf(
- bucketArgs.getStorageType()) : null,
- KeyValueUtil.getFromProtobuf(bucketArgs.getMetadataList()),
- bucketArgs.getQuotaInBytes(),
- bucketArgs.getQuotaInNamespace());
+ OmBucketArgs omBucketArgs =
+ new OmBucketArgs(bucketArgs.getVolumeName(),
bucketArgs.getBucketName(),
+ bucketArgs.hasIsVersionEnabled() ?
+ bucketArgs.getIsVersionEnabled() :
+ null, bucketArgs.hasStorageType() ?
+ StorageType.valueOf(bucketArgs.getStorageType()) :
+ null, KeyValueUtil.getFromProtobuf(bucketArgs.getMetadataList()),
+ bucketArgs.getQuotaInBytes(), bucketArgs.getQuotaInNamespace());
+ // OmBucketArgs ctor already has more arguments, so setting the default
+ // replication config separately.
+ omBucketArgs.setDefaultReplicationConfig(
+ new
DefaultReplicationConfig(bucketArgs.getDefaultReplicationConfig()));
+ return omBucketArgs;
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index b30121c..42197cc 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -37,10 +37,14 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.StorageType;
@@ -3753,4 +3757,47 @@ public abstract class TestOzoneRpcClientAbstract {
createRequiredForVersioningTest(volumeName, bucketName, keyName, true);
checkExceptedResultForVersioningTest(volumeName, bucketName, keyName, 2);
}
+
+ @Test
+ public void testSetECReplicationConfigOnBucket()
+ throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ OzoneBucket bucket = getBucket(volume);
+ ReplicationConfig currentReplicationConfig = bucket.getReplicationConfig();
+ Assert.assertEquals(
+ new StandaloneReplicationConfig(HddsProtos.ReplicationFactor.ONE),
+ currentReplicationConfig);
+ ECReplicationConfig ecReplicationConfig =
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, 1024);
+ bucket.setReplicationConfig(ecReplicationConfig);
+
+ // Get the bucket and check the updated config.
+ bucket = volume.getBucket(bucket.getName());
+
+ Assert.assertEquals(ecReplicationConfig, bucket.getReplicationConfig());
+
+ RatisReplicationConfig ratisReplicationConfig =
+ new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE);
+ bucket.setReplicationConfig(ratisReplicationConfig);
+
+ // Get the bucket and check the updated config.
+ bucket = volume.getBucket(bucket.getName());
+
+ Assert.assertEquals(ratisReplicationConfig, bucket.getReplicationConfig());
+
+ //Reset replication config back.
+ bucket.setReplicationConfig(currentReplicationConfig);
+ }
+
+ private OzoneBucket getBucket(OzoneVolume volume) throws IOException {
+ String bucketName = UUID.randomUUID().toString();
+ BucketArgs.Builder builder = BucketArgs.newBuilder();
+ builder.setVersioning(true).setDefaultReplicationConfig(
+ new DefaultReplicationConfig(
+ new
StandaloneReplicationConfig(HddsProtos.ReplicationFactor.ONE)));
+ volume.createBucket(bucketName, builder.build());
+ return volume.getBucket(bucketName);
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
index 44c3756..8660ed9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
@@ -23,6 +23,7 @@ import java.util.List;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
@@ -182,6 +183,13 @@ public class OMBucketSetPropertyRequest extends
OMClientRequest {
dbBucketInfo.getQuotaInNamespace());
}
+ DefaultReplicationConfig defaultReplicationConfig =
+ omBucketArgs.getDefaultReplicationConfig();
+ if (defaultReplicationConfig != null) {
+ // Resetting the default replication config.
+
bucketInfoBuilder.setDefaultReplicationConfig(defaultReplicationConfig);
+ }
+
bucketInfoBuilder.setCreationTime(dbBucketInfo.getCreationTime());
bucketInfoBuilder.setModificationTime(
setBucketPropertyRequest.getModificationTime());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
