This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new dc6f279 HDDS-6205. Add CLI command to display the latest Replication
Manager report (#3013)
dc6f279 is described below
commit dc6f27920c756d30fe7979872d3834abd2dc241a
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Fri Jan 28 15:50:38 2022 +0000
HDDS-6205. Add CLI command to display the latest Replication Manager report
(#3013)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 9 ++
.../scm/container/ReplicationManagerReport.java | 95 +++++++++++-
.../protocol/StorageContainerLocationProtocol.java | 9 ++
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../container/TestReplicationManagerReport.java | 49 +++++++
...inerLocationProtocolClientSideTranslatorPB.java | 17 +++
.../src/main/proto/ScmAdminProtocol.proto | 11 ++
.../interface-client/src/main/proto/hdds.proto | 16 +++
.../hdds/scm/container/ReplicationManager.java | 5 +
...inerLocationProtocolServerSideTranslatorPB.java | 16 +++
.../hdds/scm/server/SCMClientProtocolServer.java | 10 ++
.../hdds/scm/cli/ContainerOperationClient.java | 7 +
.../hdds/scm/cli/container/ContainerCommands.java | 3 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 116 +++++++++++++++
.../scm/cli/container/TestReportSubCommand.java | 159 +++++++++++++++++++++
15 files changed, 521 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index b1bf420..f1885f8 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -21,6 +21,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -320,6 +321,14 @@ public interface ScmClient extends Closeable {
boolean getReplicationManagerStatus() throws IOException;
/**
+ * Returns the latest container summary report generated by Replication
+ * Manager.
+ * @return The latest ReplicationManagerReport.
+ * @throws IOException
+ */
+ ReplicationManagerReport getReplicationManagerReport() throws IOException;
+
+ /**
* Start ContainerBalancer.
*/
boolean startContainerBalancer(Optional<Double> threshold,
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
similarity index 65%
rename from
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
rename to
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
index 21bde49..6f6caf3 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.LongAdder;
+import java.util.stream.Collectors;
/**
* This class is used by ReplicationManager. Each time ReplicationManager runs,
@@ -96,6 +97,22 @@ public class ReplicationManagerReport {
private final Map<String, List<ContainerID>> containerSample
= new ConcurrentHashMap<>();
+ public static ReplicationManagerReport fromProtobuf(
+ HddsProtos.ReplicationManagerReportProto proto) {
+ ReplicationManagerReport report = new ReplicationManagerReport();
+ report.setTimestamp(proto.getTimestamp());
+ for (HddsProtos.KeyIntValue stat : proto.getStatList()) {
+ report.setStat(stat.getKey(), stat.getValue());
+ }
+ for (HddsProtos.KeyContainerIDList sample : proto.getStatSampleList()) {
+ report.setSample(sample.getKey(), sample.getContainerList()
+ .stream()
+ .map(c -> ContainerID.getFromProtobuf(c))
+ .collect(Collectors.toList()));
+ }
+ return report;
+ }
+
public ReplicationManagerReport() {
stats = createStatsMap();
}
@@ -129,16 +146,62 @@ public class ReplicationManagerReport {
return reportTimeStamp;
}
+ /**
+ * Get the stat for the given LifeCycleState. If there is no stat available
+ * for that stat -1 is returned.
+ * @param stat The requested stat.
+ * @return The stat value or -1 if it is not present
+ */
public long getStat(HddsProtos.LifeCycleState stat) {
return getStat(stat.toString());
}
+ /**
+ * Get the stat for the given HealthState. If there is no stat available
+ * for that stat -1 is returned.
+ * @param stat The requested stat.
+ * @return The stat value or -1 if it is not present
+ */
public long getStat(HealthState stat) {
return getStat(stat.toString());
}
+ /**
+ * Returns the stat requested, or -1 if it does not exist.
+ * @param stat The request stat
+ * @return The value of the stat or -1 if it does not exist.
+ */
private long getStat(String stat) {
- return stats.get(stat).longValue();
+ LongAdder val = stats.get(stat);
+ if (val == null) {
+ return -1;
+ }
+ return val.longValue();
+ }
+
+ protected void setTimestamp(long timestamp) {
+ this.reportTimeStamp = timestamp;
+ }
+
+ protected void setStat(String stat, long value) {
+ LongAdder adder = getStatAndEnsurePresent(stat);
+ if (adder.longValue() != 0) {
+ throw new IllegalStateException(stat + " is expected to be zero");
+ }
+ adder.add(value);
+ }
+
+ protected void setSample(String stat, List<ContainerID> sample) {
+ // First get the stat, as we should not receive a sample for a stat which
+ // does not exist.
+ getStatAndEnsurePresent(stat);
+ // Now check there is not already a sample for this stat
+ List<ContainerID> existingSample = containerSample.get(stat);
+ if (existingSample != null) {
+ throw new IllegalStateException(stat
+ + " is not expected to have existing samples");
+ }
+ containerSample.put(stat, sample);
}
public List<ContainerID> getSample(HddsProtos.LifeCycleState stat) {
@@ -160,11 +223,15 @@ public class ReplicationManagerReport {
}
private void increment(String stat) {
+ getStatAndEnsurePresent(stat).increment();
+ }
+
+ private LongAdder getStatAndEnsurePresent(String stat) {
LongAdder adder = stats.get(stat);
if (adder == null) {
throw new IllegalArgumentException("Unexpected stat " + stat);
}
- adder.increment();
+ return adder;
}
private void incrementAndSample(String stat, ContainerID container) {
@@ -189,4 +256,28 @@ public class ReplicationManagerReport {
return map;
}
+ public HddsProtos.ReplicationManagerReportProto toProtobuf() {
+ HddsProtos.ReplicationManagerReportProto.Builder proto =
+ HddsProtos.ReplicationManagerReportProto.newBuilder();
+ proto.setTimestamp(getReportTimeStamp());
+
+ for (Map.Entry<String, LongAdder> e : stats.entrySet()) {
+ proto.addStat(HddsProtos.KeyIntValue.newBuilder()
+ .setKey(e.getKey())
+ .setValue(e.getValue().longValue())
+ .build());
+ }
+
+ for (Map.Entry<String, List<ContainerID>> e : containerSample.entrySet()) {
+ HddsProtos.KeyContainerIDList.Builder sample
+ = HddsProtos.KeyContainerIDList.newBuilder();
+ sample.setKey(e.getKey());
+ for (ContainerID container : e.getValue()) {
+ sample.addContainer(container.getProtobuf());
+ }
+ proto.addStatSample(sample.build());
+ }
+ return proto.build();
+ }
+
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 579f351..9f78b31 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
@@ -318,6 +319,14 @@ public interface StorageContainerLocationProtocol extends
Closeable {
boolean getReplicationManagerStatus() throws IOException;
/**
+ * Returns the latest container summary report generated by Replication
+ * Manager.
+ * @return The latest ReplicationManagerReport.
+ * @throws IOException
+ */
+ ReplicationManagerReport getReplicationManagerReport() throws IOException;
+
+ /**
* Start ContainerBalancer.
*/
boolean startContainerBalancer(Optional<Double> threshold,
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
index 9b88c6a..3c1c209 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@ -48,7 +48,8 @@ public enum SCMAction implements AuditAction {
STOP_CONTAINER_BALANCER,
GET_CONTAINER_BALANCER_STATUS,
GET_CONTAINER_WITH_PIPELINE_BATCH,
- ADD_SCM;
+ ADD_SCM,
+ GET_REPLICATION_MANAGER_REPORT;
@Override
public String getAction() {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
similarity index 67%
rename from
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
rename to
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
index 15aca61..a05f9ab 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
@@ -22,7 +22,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
/**
* Tests for the ReplicationManagerReport class.
@@ -110,4 +113,50 @@ public class TestReplicationManagerReport {
Assert.assertEquals(new ContainerID(i), sample.get(i));
}
}
+
+ @Test
+ public void testSerializeToProtoAndBack() {
+ report.setTimestamp(12345);
+ Random rand = ThreadLocalRandom.current();
+ for (HddsProtos.LifeCycleState s : HddsProtos.LifeCycleState.values()) {
+ report.setStat(s.toString(), rand.nextInt(Integer.MAX_VALUE));
+ }
+ for (ReplicationManagerReport.HealthState s :
+ ReplicationManagerReport.HealthState.values()) {
+ report.setStat(s.toString(), rand.nextInt(Integer.MAX_VALUE));
+ List<ContainerID> containers = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ containers.add(ContainerID.valueOf(rand.nextInt(Integer.MAX_VALUE)));
+ }
+ report.setSample(s.toString(), containers);
+ }
+ HddsProtos.ReplicationManagerReportProto proto = report.toProtobuf();
+ ReplicationManagerReport newReport
+ = ReplicationManagerReport.fromProtobuf(proto);
+ Assert.assertEquals(report.getReportTimeStamp(),
+ newReport.getReportTimeStamp());
+
+ for (HddsProtos.LifeCycleState s : HddsProtos.LifeCycleState.values()) {
+ Assert.assertEquals(report.getStat(s), newReport.getStat(s));
+ }
+
+ for (ReplicationManagerReport.HealthState s :
+ ReplicationManagerReport.HealthState.values()) {
+ Assert.assertTrue(report.getSample(s).equals(newReport.getSample(s)));
+ }
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testStatCannotBeSetTwice() {
+ report.setStat(HddsProtos.LifeCycleState.CLOSED.toString(), 10);
+ report.setStat(HddsProtos.LifeCycleState.CLOSED.toString(), 10);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testSampleCannotBeSetTwice() {
+ List<ContainerID> containers = new ArrayList<>();
+ containers.add(ContainerID.valueOf(1));
+ report.setSample(HddsProtos.LifeCycleState.CLOSED.toString(), containers);
+ report.setSample(HddsProtos.LifeCycleState.CLOSED.toString(), containers);
+ }
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 62139eb..b26c0da 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -64,6 +64,8 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesResponseProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportRequestProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusRequestProto;
@@ -87,6 +89,7 @@ import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
@@ -759,6 +762,20 @@ public final class
StorageContainerLocationProtocolClientSideTranslatorPB
}
@Override
+ public ReplicationManagerReport getReplicationManagerReport()
+ throws IOException {
+ ReplicationManagerReportRequestProto request =
+ ReplicationManagerReportRequestProto.newBuilder()
+ .setTraceID(TracingUtil.exportCurrentSpan())
+ .build();
+ ReplicationManagerReportResponseProto response =
+ submitRequest(Type.GetReplicationManagerReport,
+ builder -> builder.setReplicationManagerReportRequest(request))
+ .getGetReplicationManagerReportResponse();
+ return ReplicationManagerReport.fromProtobuf(response.getReport());
+ }
+
+ @Override
public boolean startContainerBalancer(
Optional<Double> threshold, Optional<Integer> iterations,
Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 313a7c8..e2d7b16 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -76,6 +76,7 @@ message ScmContainerLocationRequest {
optional QueryUpgradeFinalizationProgressRequestProto
queryUpgradeFinalizationProgressRequest = 37;
optional GetContainerCountRequestProto getContainerCountRequest = 38;
optional GetContainerReplicasRequestProto getContainerReplicasRequest = 39;
+ optional ReplicationManagerReportRequestProto
replicationManagerReportRequest = 40;
}
message ScmContainerLocationResponse {
@@ -123,6 +124,7 @@ message ScmContainerLocationResponse {
optional QueryUpgradeFinalizationProgressResponseProto
queryUpgradeFinalizationProgressResponse = 37;
optional GetContainerCountResponseProto getContainerCountResponse = 38;
optional GetContainerReplicasResponseProto getContainerReplicasResponse = 39;
+ optional ReplicationManagerReportResponseProto
getReplicationManagerReportResponse = 40;
enum Status {
OK = 1;
@@ -168,6 +170,7 @@ enum Type {
QueryUpgradeFinalizationProgress = 32;
GetContainerCount = 33;
GetContainerReplicas = 34;
+ GetReplicationManagerReport = 35;
}
/**
@@ -468,6 +471,14 @@ message ReplicationManagerStatusResponseProto {
required bool isRunning = 1;
}
+message ReplicationManagerReportRequestProto {
+ optional string traceID = 1;
+}
+
+message ReplicationManagerReportResponseProto {
+ required ReplicationManagerReportProto report = 1;
+}
+
message FinalizeScmUpgradeRequestProto {
required string upgradeClientId = 1;
}
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index bc1b35a..b55531e 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -388,3 +388,19 @@ message SCMContainerReplicaProto {
required int64 keyCount = 6;
required int64 bytesUsed = 7;
}
+
+message KeyContainerIDList {
+ required string key = 1;
+ repeated ContainerID container = 2;
+}
+
+message KeyIntValue {
+ required string key = 1;
+ optional int64 value = 2;
+}
+
+message ReplicationManagerReportProto {
+ required int64 timestamp = 1;
+ repeated KeyIntValue stat = 2;
+ repeated KeyContainerIDList statSample = 3;
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index 224b1f9..7f52a06 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -370,6 +370,11 @@ public class ReplicationManager implements SCMService {
* This in intended to be used in tests.
*/
public synchronized void processAll() {
+ if (!shouldRun()) {
+ LOG.info("Replication Manager is not ready to run until {}ms after " +
+ "safemode exit", waitTimeInMillis);
+ return;
+ }
final long start = clock.millis();
final List<ContainerInfo> containers =
containerManager.getContainers();
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 17da776..0370623 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -70,6 +70,8 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.QueryUpgradeFinalizationProgressResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesResponseProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportRequestProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMCloseContainerRequestProto;
@@ -323,6 +325,13 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
.setReplicationManagerStatusResponse(getReplicationManagerStatus(
request.getSeplicationManagerStatusRequest()))
.build();
+ case GetReplicationManagerReport:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+
.setGetReplicationManagerReportResponse(getReplicationManagerReport(
+ request.getReplicationManagerReportRequest()))
+ .build();
case StartContainerBalancer:
return ScmContainerLocationResponse.newBuilder()
.setCmdType(request.getCmdType())
@@ -731,6 +740,13 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
.setIsRunning(impl.getReplicationManagerStatus()).build();
}
+ public ReplicationManagerReportResponseProto getReplicationManagerReport(
+ ReplicationManagerReportRequestProto request) throws IOException {
+ return ReplicationManagerReportResponseProto.newBuilder()
+ .setReport(impl.getReplicationManagerReport().toProtobuf())
+ .build();
+ }
+
public StartContainerBalancerResponseProto startContainerBalancer(
StartContainerBalancerRequestProto request)
throws IOException {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 0cdb571..59e4b2b 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import
org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
@@ -733,6 +734,15 @@ public class SCMClientProtocolServer implements
}
@Override
+ public ReplicationManagerReport getReplicationManagerReport()
+ throws IOException {
+ getScm().checkAdminAccess(getRemoteUser());
+ AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
+ SCMAction.GET_REPLICATION_MANAGER_REPORT, null));
+ return scm.getReplicationManager().getContainerReport();
+ }
+
+ @Override
public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws
IOException {
// check admin authorization
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index ca4e41b..4fbabf9 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
@@ -572,6 +573,12 @@ public class ContainerOperationClient implements ScmClient
{
}
@Override
+ public ReplicationManagerReport getReplicationManagerReport()
+ throws IOException {
+ return storageContainerLocationClient.getReplicationManagerReport();
+ }
+
+ @Override
public boolean startContainerBalancer(
Optional<Double> threshold, Optional<Integer> iterations,
Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
index 00d678c..f9dfc3f 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
@@ -44,7 +44,8 @@ import picocli.CommandLine.Spec;
ListSubcommand.class,
InfoSubcommand.class,
CreateSubcommand.class,
- CloseSubcommand.class
+ CloseSubcommand.class,
+ ReportSubcommand.class
})
@MetaInfServices(SubcommandWithParent.class)
public class ContainerCommands implements Callable<Void>, SubcommandWithParent
{
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
new file mode 100644
index 0000000..89b1a11
--- /dev/null
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.container;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * This is the handler to process the container report command.
+ */
[email protected](
+ name = "report",
+ description = "Display the container summary report",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class ReportSubcommand extends ScmSubcommand {
+
+ @CommandLine.Spec
+ private CommandLine.Model.CommandSpec spec;
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ ReplicationManagerReport report = scmClient.getReplicationManagerReport();
+ outputHeader(report.getReportTimeStamp());
+ blankLine();
+ outputContainerStats(report);
+ blankLine();
+ outputContainerHealthStats(report);
+ blankLine();
+ outputContainerSamples(report);
+ }
+
+ private void outputHeader(long epochMs) {
+ Instant reportTime = Instant.ofEpochSecond(epochMs / 1000);
+ outputHeading("Container Summary Report generated at " + reportTime);
+
+ }
+
+ private void outputContainerStats(ReplicationManagerReport report) {
+ outputHeading("Container State Summary");
+ for (HddsProtos.LifeCycleState state : HddsProtos.LifeCycleState.values())
{
+ long stat = report.getStat(state);
+ if (stat != -1) {
+ output(state + ": " + stat);
+ }
+ }
+ }
+
+ private void outputContainerHealthStats(ReplicationManagerReport report) {
+ outputHeading("Container Health Summary");
+ for (ReplicationManagerReport.HealthState state
+ : ReplicationManagerReport.HealthState.values()) {
+ long stat = report.getStat(state);
+ if (stat != -1) {
+ output(state + ": " + stat);
+ }
+ }
+ }
+
+ private void outputContainerSamples(ReplicationManagerReport report) {
+ for (ReplicationManagerReport.HealthState state
+ : ReplicationManagerReport.HealthState.values()) {
+ List<ContainerID> containers = report.getSample(state);
+ if (containers.size() > 0) {
+ output("First " + ReplicationManagerReport.SAMPLE_LIMIT + " " +
+ state + " containers:");
+ output(containers
+ .stream()
+ .map(ContainerID::toString)
+ .collect(Collectors.joining(", ")));
+ blankLine();
+ }
+ }
+ }
+
+ private void blankLine() {
+ System.out.print("\n");
+ }
+
+ private void output(String s) {
+ System.out.println(s);
+ }
+
+ private void outputHeading(String s) {
+ output(s);
+ for (int i = 0; i < s.length(); i++) {
+ System.out.print("=");
+ }
+ System.out.print("\n");
+ }
+}
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java
new file mode 100644
index 0000000..5c71076
--- /dev/null
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.container;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for the Container ReportSubCommand class.
+ */
+public class TestReportSubCommand {
+
+ private ReportSubcommand cmd;
+ private static final int SEED = 10;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ @Before
+ public void setup() throws UnsupportedEncodingException {
+ cmd = new ReportSubcommand();
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @After
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ @Test
+ public void testCorrectValuesAppearInEmptyReport() throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ Mockito.when(scmClient.getReplicationManagerReport())
+ .thenAnswer(invocation -> new ReplicationManagerReport());
+
+ cmd.execute(scmClient);
+
+ for (HddsProtos.LifeCycleState state : HddsProtos.LifeCycleState.values())
{
+ Pattern p = Pattern.compile(
+ "^"+state.toString() + ": 0$", Pattern.MULTILINE);
+ Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+ }
+
+ for (ReplicationManagerReport.HealthState state :
+ ReplicationManagerReport.HealthState.values()) {
+ Pattern p = Pattern.compile(
+ "^"+state.toString() + ": 0$", Pattern.MULTILINE);
+ Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+ }
+ }
+
+ @Test
+ public void testCorrectValuesAppearInReport() throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ Mockito.when(scmClient.getReplicationManagerReport())
+ .thenAnswer(invocation -> createReport());
+
+ cmd.execute(scmClient);
+
+ int counter = SEED;
+ for (HddsProtos.LifeCycleState state : HddsProtos.LifeCycleState.values())
{
+ Pattern p = Pattern.compile(
+ "^"+state.toString() + ": " + counter + "$", Pattern.MULTILINE);
+ Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+ counter++;
+ }
+
+ counter = SEED;
+ for (ReplicationManagerReport.HealthState state :
+ ReplicationManagerReport.HealthState.values()) {
+ Pattern p = Pattern.compile(
+ "^"+state.toString() + ": " + counter + "$", Pattern.MULTILINE);
+ Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+
+ // Check the correct samples are returned
+ p = Pattern.compile(
+ "^First 100 "+ state + " containers:\n"
+ + containerList(0, counter) + "$", Pattern.MULTILINE);
+ m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+ counter++;
+ }
+ }
+
+ private ReplicationManagerReport createReport() {
+ ReplicationManagerReport report = new ReplicationManagerReport();
+
+ int counter = SEED;
+ for (HddsProtos.LifeCycleState state : HddsProtos.LifeCycleState.values())
{
+ for (int i = 0; i < counter; i++) {
+ report.increment(state);
+ }
+ counter++;
+ }
+
+ // Add samples
+ counter = SEED;
+ for (ReplicationManagerReport.HealthState state
+ : ReplicationManagerReport.HealthState.values()) {
+ for (int i = 0; i < counter; i++) {
+ report.incrementAndSample(state, ContainerID.valueOf(i));
+ }
+ counter++;
+ }
+ return report;
+ }
+
+ private String containerList(int start, int end) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = start; i < end; i++) {
+ if (i != start) {
+ sb.append(", ");
+ }
+ sb.append("#"+i);
+ }
+ return sb.toString();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
