This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 27745b54dd HDDS-6889. EC: put key command with EC replication can use
ReplicationConfig validator (#3565)
27745b54dd is described below
commit 27745b54ddb2c00c0ba2476441f990e69075b87b
Author: swamirishi <[email protected]>
AuthorDate: Tue Jul 12 19:04:26 2022 -0700
HDDS-6889. EC: put key command with EC replication can use
ReplicationConfig validator (#3565)
---
.../hdds/client/ReplicationConfigValidator.java | 4 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 7 ++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 40 +++++++++++++++++++++-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
4 files changed, 50 insertions(+), 3 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
index e95f4c7a5d..7090c76563 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.conf.PostConstruct;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.util.regex.Pattern;
@@ -53,7 +54,8 @@ public class ReplicationConfigValidator {
if (!validationRegexp.matcher(
replicationConfig.configFormat()).matches()) {
String replication = replicationConfig.getReplication();
- if (replicationConfig instanceof ECReplicationConfig) {
+ if (HddsProtos.ReplicationType.EC ==
+ replicationConfig.getReplicationType()) {
ECReplicationConfig ecConfig =
(ECReplicationConfig) replicationConfig;
replication = ecConfig.getCodec() + "-" + ecConfig.getData() +
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index a5d1dedb5c..6271d30dc4 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfigValidator;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -1155,6 +1156,12 @@ public class RpcClient implements ClientProtocol {
+ " Erasure Coded replication.");
}
}
+
+ if (replicationConfig != null) {
+ ReplicationConfigValidator validator =
+ this.conf.getObject(ReplicationConfigValidator.class);
+ validator.validate(replicationConfig);
+ }
String requestId = UUID.randomUUID().toString();
OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index b8fc543f19..280eb3573e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -149,10 +149,12 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.slf4j.event.Level.DEBUG;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.MethodSource;
/**
@@ -196,7 +198,7 @@ public abstract class TestOzoneRpcClientAbstract {
// for testZReadKeyWithUnhealthyContainerReplica.
conf.set("ozone.scm.stale.node.interval", "10s");
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(5)
+ .setNumDatanodes(14)
.setTotalPipelineNumLimit(10)
.setScmId(scmId)
.setClusterId(clusterId)
@@ -863,6 +865,42 @@ public abstract class TestOzoneRpcClientAbstract {
}
}
+ @ParameterizedTest
+ @CsvSource({"rs-3-3-1024k,false", "xor-3-5-2048k,false",
+ "rs-3-2-1024k,true", "rs-6-3-1024k,true", "rs-10-4-1024k,true"})
+ public void testPutKeyWithReplicationConfig(String replicationValue,
+ boolean isValidReplicationConfig)
+ throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+ String value = UUID.randomUUID().toString();
+ ReplicationConfig replicationConfig =
+ new ECReplicationConfig(replicationValue);
+ if (isValidReplicationConfig) {
+ OzoneOutputStream out = bucket.createKey(keyName,
+ value.getBytes(UTF_8).length, replicationConfig, new
HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ } else {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> bucket.createKey(keyName, "dummy".getBytes(UTF_8).length,
+ replicationConfig, new HashMap<>()));
+ }
+ }
+
@Test
public void testPutKey() throws IOException {
String volumeName = UUID.randomUUID().toString();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
index 68c4e7168f..1e61d5ee8f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
@@ -113,7 +113,7 @@ public class TestSecureOzoneRpcClient extends
TestOzoneRpcClient {
CertificateClientTestImpl certificateClientTest =
new CertificateClientTestImpl(conf);
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(10)
+ .setNumDatanodes(14)
.setScmId(SCM_ID)
.setClusterId(CLUSTER_ID)
.setCertificateClient(certificateClientTest)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
