This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new ebdd1f5 HDDS-5682: EC: Provide replication config option from CLI
when creating bucket. (#2639)
ebdd1f5 is described below
commit ebdd1f5e3d34c537b74fcf64c070e5fd392984ac
Author: Uma Maheswara Rao G <[email protected]>
AuthorDate: Thu Sep 16 12:56:29 2021 -0700
HDDS-5682: EC: Provide replication config option from CLI when creating
bucket. (#2639)
---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 62 ++++++++++++++++++++++
.../ozone/shell/bucket/CreateBucketHandler.java | 37 +++++++++++++
2 files changed, 99 insertions(+)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index b32ff32..8491ade 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -39,6 +39,11 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
+import org.apache.hadoop.ozone.client.io.KeyOutputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -133,11 +138,13 @@ public class TestOzoneShellHA {
numOfOMs = 3;
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
+ final int numDNs = 5;
cluster = MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
.setNumOfOzoneManagers(numOfOMs)
+ .setNumDatanodes(numDNs)
.build();
cluster.waitForClusterToBeReady();
}
@@ -855,4 +862,59 @@ public class TestOzoneShellHA {
objectStore.getVolume("vol4").deleteBucket("buck4");
objectStore.deleteVolume("vol4");
}
+
+ @Test
+ public void testCreateBucketWithECReplicationConfig() throws Exception {
+ final String volumeName = "volume100";
+ getVolume(volumeName);
+ String[] args =
+ new String[] {"bucket", "create", "/volume100/bucket0", "-rt", "EC",
+ "-r", "3-2"};
+ execute(ozoneShell, args);
+
+ OzoneVolume volume =
+ cluster.getClient().getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket("bucket0");
+ try (OzoneOutputStream out = bucket.createKey("myKey", 2000)) {
+ Assert.assertTrue(out.getOutputStream() instanceof ECKeyOutputStream);
+ }
+ }
+
+ @Test
+ public void testCreateBucketWithRatisReplicationConfig() throws Exception {
+ final String volumeName = "volume101";
+ getVolume(volumeName);
+ String[] args =
+ new String[] {"bucket", "create", "/volume101/bucket1", "-rt", "RATIS",
+ "-r", "3"};
+ execute(ozoneShell, args);
+
+ OzoneVolume volume =
+ cluster.getClient().getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket("bucket1");
+ try (OzoneOutputStream out = bucket.createKey("myKey", 2000)) {
+ Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream);
+ Assert.assertFalse(out.getOutputStream() instanceof ECKeyOutputStream);
+ }
+ }
+
+ @Test
+ public void testCreateBucketWithECReplicationConfigWithoutReplicationParam()
{
+ getVolume("volume102");
+ String[] args =
+ new String[] {"bucket", "create", "/volume102/bucket2", "-rt", "EC"};
+ try {
+ execute(ozoneShell, args);
+ Assert.fail("Must throw Exception when missing replication param");
+ } catch (Exception e) {
+ Assert.assertEquals(e.getCause().getMessage(),
+ "Replication can't be null. Replication type passed was : EC");
+ }
+ }
+
+ private void getVolume(String volumeName) {
+ String[] args = new String[] {"volume", "create",
+ "o3://" + omServiceId + "/" + volumeName};
+ execute(ozoneShell, args);
+ }
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
index 34cd876..c1ebc64 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -18,8 +18,14 @@
package org.apache.hadoop.ozone.shell.bucket;
import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -60,6 +66,17 @@ public class CreateBucketHandler extends BucketHandler {
defaultValue = "LEGACY")
private AllowedBucketLayouts allowedBucketLayout;
+ @Option(names = {"--replication", "-r"},
+ description = "Replication value. Example: 3 (for Ratis type) or 1 ( for"
+ + " standalone type). In the case of EC, pass DATA-PARITY, eg 3-2," +
+ " 6-3, 10-4")
+ private String replication;
+
+ @Option(names = {"--replicationtype", "-rt"},
+ description = "Replication type. Supported types are RATIS, STANDALONE,"
+ + " EC")
+ private String replicationType;
+
@CommandLine.Mixin
private SetSpaceQuotaOptions quotaOptions;
@@ -95,6 +112,26 @@ public class CreateBucketHandler extends BucketHandler {
}
}
+ if(replicationType!=null) {
+ if (replication != null) {
+ ReplicationConfig replicationConfig = ReplicationConfig
+ .fromTypeAndString(ReplicationType.valueOf(replicationType),
+ replication);
+ boolean isEC = replicationConfig
+ .getReplicationType() == HddsProtos.ReplicationType.EC;
+ bb.setDefaultReplicationConfig(new DefaultReplicationConfig(
+ ReplicationType.fromProto(replicationConfig.getReplicationType()),
+ isEC ?
+ null :
+
ReplicationFactor.valueOf(replicationConfig.getRequiredNodes()),
+ isEC ? (ECReplicationConfig) replicationConfig : null));
+ } else {
+ throw new IOException(
+ "Replication can't be null. Replication type passed was : "
+ + replicationType);
+ }
+ }
+
if (!Strings.isNullOrEmpty(quotaOptions.getQuotaInBytes())) {
bb.setQuotaInBytes(OzoneQuota.parseSpaceQuota(
quotaOptions.getQuotaInBytes()).getQuotaInBytes());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
