This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 6b5c01c HDDS-6184. EC: put command should create EC key if bucket is
EC (#2990)
6b5c01c is described below
commit 6b5c01c8433a771edfacc61fc958a0518131865c
Author: Uma Maheswara Rao G <[email protected]>
AuthorDate: Fri Jan 28 02:34:49 2022 -0800
HDDS-6184. EC: put command should create EC key if bucket is EC (#2990)
---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 26 +++++
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 8 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 +++++++++++++++++++++
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 42 +++++++++
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 95 +++++++++++++++++++
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 7 +-
6 files changed, 280 insertions(+), 3 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index aba9954..1a18b26 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.OzoneAdmin;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.fs.ozone.OzoneFsShell;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -41,6 +42,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
@@ -911,6 +913,30 @@ public class TestOzoneShellHA {
}
@Test
+ public void testPutKeyOnBucketWithECReplicationConfig() throws Exception {
+ final String volumeName = UUID.randomUUID().toString();
+ final String bucketName = UUID.randomUUID().toString();
+ final String keyName = UUID.randomUUID().toString();
+ getVolume(volumeName);
+ String bucketPath =
+ Path.SEPARATOR + volumeName + Path.SEPARATOR + bucketName;
+ String[] args =
+ new String[] {"bucket", "create", bucketPath, "-t", "EC", "-r",
+ "rs-3-2-1024k"};
+ execute(ozoneShell, args);
+
+ args = new String[] {"key", "put", bucketPath + Path.SEPARATOR + keyName,
+ testFilePathString};
+ execute(ozoneShell, args);
+
+ OzoneKeyDetails key =
+ cluster.getClient().getObjectStore().getVolume(volumeName)
+ .getBucket(bucketName).getKey(keyName);
+ assertEquals(HddsProtos.ReplicationType.EC,
+ key.getReplicationConfig().getReplicationType());
+ }
+
+ @Test
public void testCreateBucketWithRatisReplicationConfig() throws Exception {
final String volumeName = "volume101";
getVolume(volumeName);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
index f8a2add..ea559f8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
@@ -44,12 +44,18 @@ public final class OzoneConfigUtil {
} else {
// type is NONE, so, let's look for the bucket defaults.
if (bucketDefaultReplicationConfig != null) {
+ boolean hasECReplicationConfig = bucketDefaultReplicationConfig
+ .getType() == ReplicationType.EC && bucketDefaultReplicationConfig
+ .getEcReplicationConfig() != null;
// Since Bucket defaults are available, let's inherit
replicationConfig = ReplicationConfig.fromProto(
ReplicationType.toProto(bucketDefaultReplicationConfig.getType()),
ReplicationFactor
.toProto(bucketDefaultReplicationConfig.getFactor()),
- bucketDefaultReplicationConfig.getEcReplicationConfig().toProto());
+ hasECReplicationConfig ?
+ bucketDefaultReplicationConfig.getEcReplicationConfig()
+ .toProto() :
+ null);
} else {
// if bucket defaults also not available, then use server defaults.
replicationConfig = omDefaultReplicationConfig;
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
new file mode 100644
index 0000000..7dc07603
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests the server side replication config preference logic.
+ */
+public class TestOzoneConfigUtil {
+ private ReplicationConfig ratis3ReplicationConfig =
+ new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE);
+ private HddsProtos.ReplicationType noneType =
HddsProtos.ReplicationType.NONE;
+ private HddsProtos.ReplicationFactor zeroFactor =
+ HddsProtos.ReplicationFactor.ZERO;
+ private HddsProtos.ECReplicationConfig clientECReplicationConfig =
+ new ECReplicationConfig("rs-3-2-1024K").toProto();
+ private DefaultReplicationConfig bucketECConfig =
+ new DefaultReplicationConfig(
+ new ECReplicationConfig(clientECReplicationConfig));
+
+ /**
+ * Tests EC bucket defaults.
+ */
+ @Test
+ public void testResolveClientSideRepConfigWhenBucketHasEC() {
+ ReplicationConfig replicationConfig = OzoneConfigUtil
+ .resolveReplicationConfigPreference(noneType, zeroFactor,
+ clientECReplicationConfig, bucketECConfig,
ratis3ReplicationConfig);
+ // Client has no preference, so we should bucket defaults as we passed.
+ Assert.assertEquals(bucketECConfig.getEcReplicationConfig(),
+ replicationConfig);
+ }
+
+ /**
+ * Tests server defaults.
+ */
+ @Test
+ public void testResolveClientSideRepConfigWithNoClientAndBucketDefaults() {
+ ReplicationConfig replicationConfig = OzoneConfigUtil
+ .resolveReplicationConfigPreference(noneType, zeroFactor,
+ clientECReplicationConfig, null, ratis3ReplicationConfig);
+ // Client has no preference, no bucket defaults, so it should return server
+ // defaults.
+ Assert.assertEquals(ratis3ReplicationConfig, replicationConfig);
+ }
+
+ /**
+ * Tests client preference of EC.
+ */
+ @Test
+ public void testResolveClientSideRepConfigWhenClientPassEC() {
+ ReplicationConfig replicationConfig = OzoneConfigUtil
+ .resolveReplicationConfigPreference(HddsProtos.ReplicationType.EC,
+ zeroFactor, clientECReplicationConfig, null,
+ ratis3ReplicationConfig);
+ // Client has preference of type EC, no bucket defaults, so it should
return
+ // client preference.
+ Assert.assertEquals(new ECReplicationConfig("rs-3-2-1024K"),
+ replicationConfig);
+ }
+
+ /**
+ * Tests bucket ratis defaults.
+ */
+ @Test
+ public void testResolveClientSideRepConfigWhenBucketHasEC3() {
+ DefaultReplicationConfig ratisBucketDefaults =
+ new DefaultReplicationConfig(ReplicationType.RATIS,
+ ReplicationFactor.THREE);
+ ReplicationConfig replicationConfig = OzoneConfigUtil
+ .resolveReplicationConfigPreference(noneType, zeroFactor,
+ clientECReplicationConfig, ratisBucketDefaults,
+ ratis3ReplicationConfig);
+ // Client has no preference of type and bucket has ratis defaults, so it
+ // should return ratis.
+ Assert.assertEquals(ratisBucketDefaults.getType().name(),
+ replicationConfig.getReplicationType().name());
+ Assert.assertEquals(ratisBucketDefaults.getFactor(),
+ ReplicationFactor.valueOf(replicationConfig.getRequiredNodes()));
+ }
+
+}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
index 6e94553..b632aae 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
@@ -147,4 +147,46 @@ public final class OzoneClientUtils {
config.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT)),
replication, config);
}
+
+ /**
+ * Gets the client side replication config by checking user passed values vs
+ * client configured values.
+ * @param userPassedType - User provided replication type.
+ * @param userPassedReplication - User provided replication.
+ * @param clientSideConfig - Client side configuration.
+ * @return ReplicationConfig.
+ */
+ public static ReplicationConfig validateAndGetClientReplicationConfig(
+ ReplicationType userPassedType, String userPassedReplication,
+ OzoneConfiguration clientSideConfig) {
+ // Priority 1: User passed replication config values.
+ // Priority 2: Client side configured replication config values.
+ /* if above two are not available, we should just return null and clients
+ can pass null replication config to server. Now server will take the
+ decision of finding the replication config( either from bucket defaults
+ or server defaults). */
+ ReplicationType clientReplicationType = userPassedType;
+ String clientReplication = userPassedReplication;
+ String clientConfiguredDefaultType =
+ clientSideConfig.get(OZONE_REPLICATION_TYPE);
+ if (userPassedType == null && clientConfiguredDefaultType != null) {
+ clientReplicationType =
+ ReplicationType.valueOf(clientConfiguredDefaultType);
+ }
+
+ String clientConfiguredDefaultReplication =
+ clientSideConfig.get(OZONE_REPLICATION);
+ if (userPassedReplication == null
+ && clientConfiguredDefaultReplication != null) {
+ clientReplication = clientConfiguredDefaultReplication;
+ }
+
+ // if clientReplicationType or clientReplication is null, then we just pass
+ // replication config as null, so that server will take decision.
+ if (clientReplicationType == null || clientReplication == null) {
+ return null;
+ }
+ return ReplicationConfig
+ .parse(clientReplicationType, clientReplication, clientSideConfig);
+ }
}
diff --git
a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java
index 46ab0bf..f610b75 100644
---
a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java
+++
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.fs.ozone;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.junit.Assert;
import org.junit.Test;
@@ -128,4 +130,97 @@ public class TestOzoneClientUtils {
// Configured value is ratis THREE
Assert.assertEquals(ratis3ReplicationConfig, replicationConfig);
}
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user passed valid config
+ * values.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenValidUserPassedValues() {
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(ReplicationType.RATIS, "1",
+ new OzoneConfiguration());
+ // Configured value is ratis ONE
+ Assert.assertEquals(ratis1ReplicationConfig, replicationConfig);
+ }
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user passed null values.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenValidUserPassedNullValues() {
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(null, null,
+ new OzoneConfiguration());
+ Assert.assertNull(replicationConfig);
+ }
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user passed null values
+ * but client config has valid values.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenValidConfigValues() {
+ OzoneConfiguration clientSideConfig = new OzoneConfiguration();
+ clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, "EC");
+ clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024K");
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(null, null, clientSideConfig);
+ Assert.assertEquals(ecReplicationConfig, replicationConfig);
+ }
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user passed null values
+ * but client config has valid values.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenNullTypeFromUser() {
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(null, "3",
+ new OzoneConfiguration());
+ Assert.assertNull(replicationConfig);
+ }
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user passed null
+ * replication but valid type.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenNullReplicationFromUser() {
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(ReplicationType.EC, null,
+ new OzoneConfiguration());
+ Assert.assertNull(replicationConfig);
+ }
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user pass null values but
+ * config has only replication configured.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenNullTypeConfigValues() {
+ OzoneConfiguration clientSideConfig = new OzoneConfiguration();
+ clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024K");
+ //By default config values are null. Let's don't set type to keep it as
+ // null.
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(null, null, clientSideConfig);
+ Assert.assertNull(replicationConfig);
+ }
+
+ /**
+ * Tests validateAndGetClientReplicationConfig with user pass null values but
+ * config has only type configured.
+ */
+ @Test
+ public void testValidateAndGetRepConfWhenNullReplicationConfigValues() {
+ OzoneConfiguration clientSideConfig = new OzoneConfiguration();
+ clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, "EC");
+ //By default config values are null. Let's don't set replication to keep it
+ // as null.
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(null, null, clientSideConfig);
+ Assert.assertNull(replicationConfig);
+ }
+
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
index c32fc45..0c4e33c 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -27,6 +27,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.ozone.OzoneClientUtils;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.io.IOUtils;
@@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.shell.OzoneAddress;
import org.apache.commons.codec.digest.DigestUtils;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
@@ -83,8 +85,9 @@ public class PutKeyHandler extends KeyHandler {
}
}
- ReplicationConfig replicationConfig =
- ReplicationConfig.parse(replicationType, replication, getConf());
+ ReplicationConfig replicationConfig = OzoneClientUtils
+ .validateAndGetClientReplicationConfig(replicationType, replication,
+ getConf());
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
OzoneBucket bucket = vol.getBucket(bucketName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
