This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 722c444 HDDS-6250. EC: Add replica index to the output in the
container info command (#3041)
722c444 is described below
commit 722c444aea306b41cdbf5acbf13ecb8d51178746
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Mon Feb 7 18:56:11 2022 +0000
HDDS-6250. EC: Add replica index to the output in the container info
command (#3041)
---
.../hdds/scm/container/ContainerReplicaInfo.java | 14 ++++-
.../scm/container/TestContainerReplicaInfo.java | 31 +++++++++
.../interface-client/src/main/proto/hdds.proto | 1 +
.../hdds/scm/server/SCMClientProtocolServer.java | 7 +--
.../hdds/scm/cli/container/InfoSubcommand.java | 3 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 73 +++++++++++++++++++---
6 files changed, 117 insertions(+), 12 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java
index b30dff7..5a81f6b 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java
@@ -34,6 +34,7 @@ public final class ContainerReplicaInfo {
private long sequenceId;
private long keyCount;
private long bytesUsed;
+ private int replicaIndex = -1;
public static ContainerReplicaInfo fromProto(
HddsProtos.SCMContainerReplicaProto proto) {
@@ -45,7 +46,9 @@ public final class ContainerReplicaInfo {
.setPlaceOfBirth(UUID.fromString(proto.getPlaceOfBirth()))
.setSequenceId(proto.getSequenceID())
.setKeyCount(proto.getKeyCount())
- .setBytesUsed(proto.getBytesUsed());
+ .setBytesUsed(proto.getBytesUsed())
+ .setReplicaIndex(
+ proto.hasReplicaIndex() ? (int)proto.getReplicaIndex() : -1);
return builder.build();
}
@@ -80,6 +83,10 @@ public final class ContainerReplicaInfo {
return bytesUsed;
}
+ public int getReplicaIndex() {
+ return replicaIndex;
+ }
+
/**
* Builder for ContainerReplicaInfo class.
*/
@@ -122,6 +129,11 @@ public final class ContainerReplicaInfo {
return this;
}
+ public Builder setReplicaIndex(int replicaIndex) {
+ subject.replicaIndex = replicaIndex;
+ return this;
+ }
+
public ContainerReplicaInfo build() {
return subject;
}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReplicaInfo.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReplicaInfo.java
index 195baca..4f63ea7 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReplicaInfo.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReplicaInfo.java
@@ -55,5 +55,36 @@ public class TestContainerReplicaInfo {
proto.getDatanodeDetails()), info.getDatanodeDetails());
Assert.assertEquals(proto.getSequenceID(), info.getSequenceId());
Assert.assertEquals(proto.getState(), info.getState());
+ // If replicaIndex is not in the proto, then -1 should be returned
+ Assert.assertEquals(-1, info.getReplicaIndex());
+ }
+
+ @Test
+ public void testObjectCreatedFromProtoWithReplicaIndedx() {
+ HddsProtos.SCMContainerReplicaProto proto =
+ HddsProtos.SCMContainerReplicaProto.newBuilder()
+ .setKeyCount(10)
+ .setBytesUsed(12345)
+ .setContainerID(567)
+ .setPlaceOfBirth(UUID.randomUUID().toString())
+ .setSequenceID(5)
+ .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()
+ .getProtoBufMessage())
+ .setState("OPEN")
+ .setReplicaIndex(4)
+ .build();
+
+ ContainerReplicaInfo info = ContainerReplicaInfo.fromProto(proto);
+
+ Assert.assertEquals(proto.getContainerID(), info.getContainerID());
+ Assert.assertEquals(proto.getBytesUsed(), info.getBytesUsed());
+ Assert.assertEquals(proto.getKeyCount(), info.getKeyCount());
+ Assert.assertEquals(proto.getPlaceOfBirth(),
+ info.getPlaceOfBirth().toString());
+ Assert.assertEquals(DatanodeDetails.getFromProtoBuf(
+ proto.getDatanodeDetails()), info.getDatanodeDetails());
+ Assert.assertEquals(proto.getSequenceID(), info.getSequenceId());
+ Assert.assertEquals(proto.getState(), info.getState());
+ Assert.assertEquals(4, info.getReplicaIndex());
}
}
\ No newline at end of file
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index ed296dc..02488e2 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -406,6 +406,7 @@ message SCMContainerReplicaProto {
required int64 sequenceID = 5;
required int64 keyCount = 6;
required int64 bytesUsed = 7;
+ optional int64 replicaIndex = 8;
}
message KeyContainerIDList {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 332df77..c95612e 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -29,7 +29,6 @@ import com.google.protobuf.ProtocolMessageEnum;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -258,8 +257,7 @@ public class SCMClientProtocolServer implements
if (pipeline == null) {
pipeline = scm.getPipelineManager().createPipeline(
- new StandaloneReplicationConfig(ReplicationConfig
- .getLegacyFactor(container.getReplicationConfig())),
+ container.getReplicationConfig(),
scm.getContainerManager()
.getContainerReplicas(cid).stream()
.map(ContainerReplica::getDatanodeDetails)
@@ -306,7 +304,8 @@ public class SCMClientProtocolServer implements
.setBytesUsed(r.getBytesUsed())
.setPlaceOfBirth(r.getOriginDatanodeId().toString())
.setKeyCount(r.getKeyCount())
- .setSequenceID(r.getSequenceId()).build()
+ .setSequenceID(r.getSequenceId())
+ .setReplicaIndex(r.getReplicaIndex()).build()
);
}
return results;
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
index 7b0daa6..25ed0e5 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
@@ -119,6 +119,9 @@ public class InfoSubcommand extends ScmSubcommand {
private static String buildReplicaDetails(ContainerReplicaInfo replica) {
StringBuilder sb = new StringBuilder();
sb.append("State: " + replica.getState() + ";");
+ if (replica.getReplicaIndex() != -1) {
+ sb.append(" ReplicaIndex: " + replica.getReplicaIndex() + ";");
+ }
sb.append(" Origin: " + replica.getPlaceOfBirth().toString() + ";");
sb.append(" Location: "
+ buildDatanodeDetails(replica.getDatanodeDetails()));
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java
index ad43f9e..2ea34b9 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.cli.container;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -81,8 +82,19 @@ public class TestInfoSubCommand {
@Test
public void testReplicasIncludedInOutput() throws Exception {
+ testReplicaIncludedInOutput(false);
+ }
+
+ @Test
+ public void testReplicaIndexInOutput() throws Exception {
+ testReplicaIncludedInOutput(true);
+ }
+
+
+ private void testReplicaIncludedInOutput(boolean includeIndex)
+ throws IOException {
Mockito.when(scmClient.getContainerReplicas(anyLong()))
- .thenReturn(getReplicas());
+ .thenReturn(getReplicas(includeIndex));
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("1");
@@ -94,7 +106,7 @@ public class TestInfoSubCommand {
.filter(m -> m.getRenderedMessage().matches("(?s)^Replicas:.*"))
.collect(Collectors.toList());
Assert.assertEquals(1, replica.size());
-
+
// Ensure each DN UUID is mentioned in the message:
for (DatanodeDetails dn : datanodes) {
Pattern pattern = Pattern.compile(".*" + dn.getUuid().toString() + ".*",
@@ -102,6 +114,11 @@ public class TestInfoSubCommand {
Matcher matcher = pattern.matcher(replica.get(0).getRenderedMessage());
Assert.assertTrue(matcher.matches());
}
+ // Ensure ReplicaIndex is not mentioned as it was not passed in the proto:
+ Pattern pattern = Pattern.compile(".*ReplicaIndex.*",
+ Pattern.DOTALL);
+ Matcher matcher = pattern.matcher(replica.get(0).getRenderedMessage());
+ Assert.assertEquals(includeIndex, matcher.matches());
}
@Test
@@ -151,7 +168,20 @@ public class TestInfoSubCommand {
@Test
public void testReplicasOutputWithJson() throws IOException {
Mockito.when(scmClient.getContainerReplicas(anyLong()))
- .thenReturn(getReplicas());
+ .thenReturn(getReplicas(true));
+ testJsonOutput();
+ }
+
+ @Test
+ public void testECContainerReplicasOutputWithJson() throws IOException {
+ Mockito.when(scmClient.getContainerReplicas(anyLong()))
+ .thenReturn(getReplicas(true));
+ Mockito.when(scmClient.getContainerWithPipeline(anyLong()))
+ .thenReturn(getECContainerWithPipeline());
+ testJsonOutput();
+ }
+
+ private void testJsonOutput() throws IOException {
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("1", "--json");
@@ -169,20 +199,29 @@ public class TestInfoSubCommand {
Matcher matcher = pattern.matcher(json);
Assert.assertTrue(matcher.matches());
}
+ Pattern pattern = Pattern.compile(".*replicaIndex.*",
+ Pattern.DOTALL);
+ Matcher matcher = pattern.matcher(json);
+ Assert.assertTrue(matcher.matches());
}
- private List<ContainerReplicaInfo> getReplicas() {
+
+ private List<ContainerReplicaInfo> getReplicas(boolean includeIndex) {
List<ContainerReplicaInfo> replicas = new ArrayList<>();
for (DatanodeDetails dn : datanodes) {
- ContainerReplicaInfo container = new ContainerReplicaInfo.Builder()
+ ContainerReplicaInfo.Builder container
+ = new ContainerReplicaInfo.Builder()
.setContainerID(1)
.setBytesUsed(1234)
.setState("CLOSED")
.setPlaceOfBirth(dn.getUuid())
.setDatanodeDetails(dn)
.setKeyCount(1)
- .setSequenceId(1).build();
- replicas.add(container);
+ .setSequenceId(1);
+ if (includeIndex) {
+ container.setReplicaIndex(4);
+ }
+ replicas.add(container.build());
}
return replicas;
}
@@ -207,6 +246,26 @@ public class TestInfoSubCommand {
return new ContainerWithPipeline(container, pipeline);
}
+ private ContainerWithPipeline getECContainerWithPipeline() {
+ Pipeline pipeline = new Pipeline.Builder()
+ .setState(Pipeline.PipelineState.CLOSED)
+ .setReplicationConfig(new ECReplicationConfig(3, 2))
+ .setId(PipelineID.randomId())
+ .setNodes(datanodes)
+ .build();
+
+ ContainerInfo container = new ContainerInfo.Builder()
+ .setSequenceId(1)
+ .setPipelineID(pipeline.getId())
+ .setUsedBytes(1234)
+ .setReplicationConfig(new RatisReplicationConfig(THREE))
+ .setNumberOfKeys(1)
+ .setState(CLOSED)
+ .build();
+
+ return new ContainerWithPipeline(container, pipeline);
+ }
+
private List<DatanodeDetails> createDatanodeDetails(int count) {
List<DatanodeDetails> dns = new ArrayList<>();
for (int i = 0; i < count; i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
