This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 550eadc HDDS-6308. EC: Handle Replication Factor to consider EC
Config in Recon UI (#3078)
550eadc is described below
commit 550eadc9704dcc6abf9abbca6227f1d1eb1c758e
Author: Uma Maheswara Rao G <[email protected]>
AuthorDate: Thu Feb 17 07:40:54 2022 -0800
HDDS-6308. EC: Handle Replication Factor to consider EC Config in Recon UI
(#3078)
---
.../hadoop/hdds/client/ECReplicationConfig.java | 10 ++++++++
.../hadoop/hdds/client/RatisReplicationConfig.java | 5 ++++
.../hadoop/hdds/client/ReplicationConfig.java | 5 ++++
.../hdds/client/StandaloneReplicationConfig.java | 5 ++++
.../hadoop/hdds/client/TestReplicationConfig.java | 29 ++++++++++++++++++++++
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +---
.../ozone/recon/api/types/DatanodePipeline.java | 14 ++++++-----
.../ozone/recon/api/types/PipelineMetadata.java | 19 +++++++-------
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +++---
11 files changed, 77 insertions(+), 26 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
index e48db41..5459674 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
@@ -31,6 +31,8 @@ import java.util.stream.Collectors;
*/
public class ECReplicationConfig implements ReplicationConfig {
+ public static final String EC_REPLICATION_PARAMS_DELIMITER = "-";
+
/**
* Enum defining the allowed list of ECCodecs.
*/
@@ -141,6 +143,14 @@ public class ECReplicationConfig implements
ReplicationConfig {
return data + parity;
}
+ @Override
+ public String getReplication() {
+ return getCodec() + EC_REPLICATION_PARAMS_DELIMITER
+ + getData() + EC_REPLICATION_PARAMS_DELIMITER
+ + getParity() + EC_REPLICATION_PARAMS_DELIMITER
+ + getEcChunkSize();
+ }
+
public HddsProtos.ECReplicationConfig toProto() {
return HddsProtos.ECReplicationConfig.newBuilder()
.setData(data)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
index 43f5eba..c9e162d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
@@ -62,6 +62,11 @@ public class RatisReplicationConfig implements
ReplicatedReplicationConfig {
}
@Override
+ public String getReplication() {
+ return String.valueOf(replicationFactor);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) {
return true;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
index c031ed6..9efb5ec 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
@@ -195,4 +195,9 @@ public interface ReplicationConfig {
*/
int getRequiredNodes();
+ /**
+ * Returns the replication option in string format.
+ */
+ String getReplication();
+
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
index 840ba18..4cf111d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
@@ -48,6 +48,11 @@ public class StandaloneReplicationConfig implements
}
@Override
+ public String getReplication() {
+ return String.valueOf(this.replicationFactor);
+ }
+
+ @Override
public ReplicationType getReplicationType() {
return ReplicationType.STAND_ALONE;
}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java
index 7346616..ed8c8f7 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -160,6 +161,34 @@ public class TestReplicationConfig {
}
@Test
+ public void testECReplicationConfigGetReplication() {
+ assumeECType();
+ HddsProtos.ECReplicationConfig proto =
+ HddsProtos.ECReplicationConfig.newBuilder().setCodec(codec)
+ .setData(data).setParity(parity).setEcChunkSize(chunkSize).build();
+
+ ReplicationConfig config =
+ ReplicationConfig.fromProto(ReplicationType.EC, null, proto);
+
+ Assert.assertEquals(EcCodec.valueOf(
+ codec) + ECReplicationConfig.EC_REPLICATION_PARAMS_DELIMITER
+ + data + ECReplicationConfig.EC_REPLICATION_PARAMS_DELIMITER
+ + parity + ECReplicationConfig.EC_REPLICATION_PARAMS_DELIMITER
+ + chunkSize, config.getReplication());
+ }
+
+ @Test
+ public void testReplicationConfigGetReplication() {
+ assumeRatisOrStandaloneType();
+ final ReplicationConfig replicationConfig = ReplicationConfig
+ .fromTypeAndFactor(
+ org.apache.hadoop.hdds.client.ReplicationType.valueOf(type),
+ org.apache.hadoop.hdds.client.ReplicationFactor.valueOf(factor));
+
+ Assert.assertEquals(factor, replicationConfig.getReplication());
+ }
+
+ @Test
public void fromJavaObjects() {
assumeRatisOrStandaloneType();
final ReplicationConfig replicationConfig =
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
index d475be4..0595bbf 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.recon.api;
-import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory;
@@ -105,9 +104,7 @@ public class PipelineEndpoint {
.setDatanodes(datanodes)
.setDuration(duration)
.setStatus(pipeline.getPipelineState())
- .setReplicationFactor(ReplicationConfig
-
.getLegacyFactor(pipeline.getReplicationConfig()).getNumber())
- .setReplicationType(pipeline.getType().toString());
+ .setReplicationConfig(pipeline.getReplicationConfig());
// If any metrics service providers like Prometheus
// is configured, then query it for metrics and populate
// leader election count and last leader election time
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
index 88a2989..e302cad 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
@@ -26,13 +26,15 @@ import java.util.UUID;
*/
public class DatanodePipeline {
private UUID pipelineID;
- private ReplicationConfig repConfig;
+ private String replicationType;
+ private String replicationFactor;
private String leaderNode;
- public DatanodePipeline(UUID pipelineID, ReplicationConfig repConfig,
+ public DatanodePipeline(UUID pipelineID, ReplicationConfig replicationConfig,
String leaderNode) {
this.pipelineID = pipelineID;
- this.repConfig = repConfig;
+ this.replicationType = replicationConfig.getReplicationType().toString();
+ this.replicationFactor = replicationConfig.getReplication();
this.leaderNode = leaderNode;
}
@@ -41,11 +43,11 @@ public class DatanodePipeline {
}
public String getReplicationType() {
- return repConfig.getReplicationType().name();
+ return replicationType;
}
- public ReplicationConfig getReplicationConfig() {
- return repConfig;
+ public String getReplicationFactor() {
+ return replicationFactor;
}
public String getLeaderNode() {
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/PipelineMetadata.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/PipelineMetadata.java
index 7af171f..e7facbf 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/PipelineMetadata.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/PipelineMetadata.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.recon.api.types;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState;
import javax.xml.bind.annotation.XmlAccessType;
@@ -57,8 +58,10 @@ public final class PipelineMetadata {
@XmlElement(name = "replicationType")
private String replicationType;
+ // TODO: name can be changed to just "replication". Currently EC replication
+ // also showed with below parameter but in String format.
@XmlElement(name = "replicationFactor")
- private int replicationFactor;
+ private String replicationFactor;
@XmlElement(name = "containers")
private int containers;
@@ -95,7 +98,7 @@ public final class PipelineMetadata {
return replicationType;
}
- public int getReplicationFactor() {
+ public String getReplicationFactor() {
return replicationFactor;
}
@@ -139,7 +142,7 @@ public final class PipelineMetadata {
private long duration;
private long leaderElections;
private String replicationType;
- private int replicationFactor;
+ private String replicationFactor;
private int containers;
public Builder() {
@@ -200,13 +203,9 @@ public final class PipelineMetadata {
return this;
}
- public Builder setReplicationType(String replicationType) {
- this.replicationType = replicationType;
- return this;
- }
-
- public Builder setReplicationFactor(int replicationFactor) {
- this.replicationFactor = replicationFactor;
+ public Builder setReplicationConfig(ReplicationConfig replicationConfig) {
+ this.replicationType = replicationConfig.getReplicationType().toString();
+ this.replicationFactor = replicationConfig.getReplication();
return this;
}
diff --git
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
index 62093d6..1781002 100644
---
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
+++
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
@@ -82,7 +82,7 @@ interface IDatanode {
interface IPipeline {
pipelineID: string;
replicationType: string;
- replicationFactor: number;
+ replicationFactor: string;
leaderNode: string;
}
diff --git
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
index 342a8bd..6810cba 100644
---
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
+++
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
@@ -44,7 +44,7 @@ interface IPipelineResponse {
lastLeaderElection: number;
duration: number;
leaderElections: number;
- replicationFactor: number;
+ replicationFactor: string;
containers: number;
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
index 80fd59f..8ea440b 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
@@ -414,9 +414,8 @@ public class TestEndpoints extends AbstractReconSqlDBTest {
Assert.assertEquals(1, datanodeMetadata.getPipelines().size());
Assert.assertEquals(pipelineId,
datanodeMetadata.getPipelines().get(0).getPipelineID().toString());
- Assert.assertEquals(pipeline.getReplicationConfig().getRequiredNodes(),
- datanodeMetadata.getPipelines().get(0).getReplicationConfig()
- .getRequiredNodes());
+ Assert.assertEquals(pipeline.getReplicationConfig().getReplication(),
+ datanodeMetadata.getPipelines().get(0).getReplicationFactor());
Assert.assertEquals(pipeline.getType().toString(),
datanodeMetadata.getPipelines().get(0).getReplicationType());
Assert.assertEquals(pipeline.getLeaderNode().getHostName(),
@@ -523,7 +522,7 @@ public class TestEndpoints extends AbstractReconSqlDBTest {
Assert.assertEquals(1, pipelineMetadata.getDatanodes().size());
Assert.assertEquals(pipeline.getType().toString(),
pipelineMetadata.getReplicationType());
- Assert.assertEquals(pipeline.getReplicationConfig().getRequiredNodes(),
+ Assert.assertEquals(pipeline.getReplicationConfig().getReplication(),
pipelineMetadata.getReplicationFactor());
Assert.assertEquals(datanodeDetails.getHostName(),
pipelineMetadata.getLeaderNode());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
