This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e5318a38a9 HDDS-6841. EC: Validate the server default configuration on
Ozone manager startup (#3498)
e5318a38a9 is described below
commit e5318a38a9d60beec05ef0c1b84c5712f3f329f6
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Tue Jun 14 12:29:47 2022 -0700
HDDS-6841. EC: Validate the server default configuration on Ozone manager
startup (#3498)
---
.../apache/hadoop/hdds/client/ECReplicationConfig.java | 5 +++++
.../apache/hadoop/hdds/client/RatisReplicationConfig.java | 5 +++++
.../org/apache/hadoop/hdds/client/ReplicationConfig.java | 5 ++++-
.../hadoop/hdds/client/ReplicationConfigValidator.java | 14 +++++++++++---
.../hadoop/hdds/client/StandaloneReplicationConfig.java | 5 +++++
.../hdds/client/TestReplicationConfigValidator.java | 15 ++++++++++++++-
.../java/org/apache/hadoop/ozone/om/OzoneManager.java | 7 +++++++
7 files changed, 51 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
index deb17cdcec..daf1d53151 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
@@ -204,4 +204,9 @@ public class ECReplicationConfig implements
ReplicationConfig {
+ ", ecChunkSize=" + ecChunkSize
+ ", codec=" + codec + '}';
}
+
+ @Override
+ public String configFormat() {
+ return HddsProtos.ReplicationType.EC.name() + "/" + data + "-" + parity;
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
index b0c1dd2b6a..3cf2d190dc 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
@@ -119,4 +119,9 @@ public final class RatisReplicationConfig
public String toString() {
return REPLICATION_TYPE.name() + "/" + replicationFactor;
}
+
+ @Override
+ public String configFormat() {
+ return toString();
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
index e3e69b44f3..6135883158 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
@@ -192,7 +192,8 @@ public interface ReplicationConfig {
replicationConfig = fromTypeAndFactor(type, factor);
break;
case EC:
- return new ECReplicationConfig(replication);
+ replicationConfig = new ECReplicationConfig(replication);
+ break;
default:
throw new RuntimeException("Replication type" + type + " can not"
+ "be parsed.");
@@ -220,4 +221,6 @@ public interface ReplicationConfig {
*/
String getReplication();
+ String configFormat();
+
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
index 5041f6a37e..e95f4c7a5d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
@@ -50,10 +50,18 @@ public class ReplicationConfigValidator {
public ReplicationConfig validate(ReplicationConfig replicationConfig) {
if (validationRegexp != null) {
- if (!validationRegexp.matcher(replicationConfig.toString()).matches()) {
+ if (!validationRegexp.matcher(
+ replicationConfig.configFormat()).matches()) {
+ String replication = replicationConfig.getReplication();
+ if (replicationConfig instanceof ECReplicationConfig) {
+ ECReplicationConfig ecConfig =
+ (ECReplicationConfig) replicationConfig;
+ replication = ecConfig.getCodec() + "-" + ecConfig.getData() +
+ "-" + ecConfig.getParity() + "-{CHUNK_SIZE}";
+ }
throw new IllegalArgumentException("Invalid replication config " +
- replicationConfig + ". Replication config should match the "
- + validationPattern + " pattern.");
+ "for type " + replicationConfig.getReplicationType() +
+ " and replication " + replication);
}
}
return replicationConfig;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
index 7befffe6b9..e9d77e9beb 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
@@ -121,4 +121,9 @@ public final class StandaloneReplicationConfig implements
public String toString() {
return REPLICATION_TYPE + "/" + replicationFactor;
}
+
+ @Override
+ public String configFormat() {
+ return toString();
+ }
}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
index 320069c060..bbdc78c94a 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.client;
import org.apache.hadoop.hdds.conf.InMemoryConfiguration;
import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
+import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.Test;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
@@ -36,11 +37,23 @@ public class TestReplicationConfigValidator {
final ReplicationConfigValidator validator =
config.getObject(ReplicationConfigValidator.class);
+ String ecConfig1 = "rs-3-2-1024k";
+ String ecConfig2 = "xor-6-3-2048k";
+ //Supported data-parity are 3-2,6-3,10-4
+ String invalidEcConfig1 = "xor-6-4-1024k";
validator.validate(RatisReplicationConfig.getInstance(THREE));
validator.validate(RatisReplicationConfig.getInstance(ONE));
validator.validate(StandaloneReplicationConfig.getInstance(THREE));
validator.validate(StandaloneReplicationConfig.getInstance(ONE));
+ validator.validate(new ECReplicationConfig(ecConfig1));
+ validator.validate(new ECReplicationConfig(ecConfig2));
+ try {
+ validator.validate(new ECReplicationConfig(invalidEcConfig1));
+ } catch (IllegalArgumentException ex) {
+ GenericTestUtils.assertExceptionContains("Invalid replication " +
+ "config for type EC and replication xor-6-4-{CHUNK_SIZE}", ex);
+ }
}
@@ -71,4 +84,4 @@ public class TestReplicationConfigValidator {
() -> validator.validate(RatisReplicationConfig.getInstance(ONE)));
}
-}
\ No newline at end of file
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 20ce38d406..fd413b573c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -389,6 +389,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
private final boolean grpcBlockTokenEnabled;
private final boolean useRatisForReplication;
private final String defaultBucketLayout;
+ private ReplicationConfig defaultReplicationConfig;
private boolean isS3MultiTenancyEnabled;
@@ -516,6 +517,8 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
BucketLayout.OBJECT_STORE + ", " + BucketLayout.LEGACY + ".");
}
+ // Validates the default server-side replication configs.
+ this.defaultReplicationConfig = getDefaultReplicationConfig();
InetSocketAddress omNodeRpcAddr = omNodeDetails.getRpcAddress();
omRpcAddressTxt = new Text(omNodeDetails.getRpcAddressString());
@@ -4172,6 +4175,10 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
public ReplicationConfig getDefaultReplicationConfig() {
+ if (this.defaultReplicationConfig != null) {
+ return this.defaultReplicationConfig;
+ }
+
final String replication = configuration.getTrimmed(
OZONE_SERVER_DEFAULT_REPLICATION_KEY,
OZONE_SERVER_DEFAULT_REPLICATION_DEFAULT);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
