This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1c9f1051ba HDDS-9648. Create API to fetch info about a single datanode
(#5856)
1c9f1051ba is described below
commit 1c9f1051bada446a2ccf3e0aa83c6b046cfe305b
Author: Tejaskriya <[email protected]>
AuthorDate: Fri Jan 19 21:36:27 2024 +0530
HDDS-9648. Create API to fetch info about a single datanode (#5856)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 9 ++++++++
.../protocol/StorageContainerLocationProtocol.java | 3 +++
...inerLocationProtocolClientSideTranslatorPB.java | 16 +++++++++++++
.../src/main/proto/ScmAdminProtocol.proto | 15 ++++++++++--
...inerLocationProtocolServerSideTranslatorPB.java | 20 ++++++++++++++++
.../hdds/scm/server/SCMClientProtocolServer.java | 22 ++++++++++++++++++
.../hdds/scm/cli/ContainerOperationClient.java | 6 +++++
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 14 +++++++----
.../scm/cli/datanode/TestListInfoSubcommand.java | 27 ++++++++++++++++++++++
9 files changed, 126 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index b03cead27e..120535405e 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -39,6 +39,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.UUID;
/**
* The interface to call into underlying container layer.
@@ -194,6 +195,14 @@ public interface ScmClient extends Closeable {
HddsProtos.NodeState nodeState, HddsProtos.QueryScope queryScope,
String poolName) throws IOException;
+ /**
+ * Returns a node with the given UUID.
+ * @param uuid - datanode uuid string
+ * @return A nodes that matches the requested UUID.
+ * @throws IOException
+ */
+ HddsProtos.Node queryNode(UUID uuid) throws IOException;
+
/**
* Allows a list of hosts to be decommissioned. The hosts are identified
* by their hostname and optionally port in the format foo.com:port.
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index b587cc924b..be0f41b622 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -44,6 +44,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.UUID;
/**
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
@@ -232,6 +233,8 @@ public interface StorageContainerLocationProtocol extends
Closeable {
HddsProtos.NodeState state, HddsProtos.QueryScope queryScope,
String poolName, int clientVersion) throws IOException;
+ HddsProtos.Node queryNode(UUID uuid) throws IOException;
+
List<DatanodeAdminError> decommissionNodes(List<String> nodes)
throws IOException;
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 330cfae30b..eb3f419e48 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -89,6 +89,8 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMCloseContainerResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartMaintenanceNodesRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartMaintenanceNodesResponseProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SingleNodeQueryRequestProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SingleNodeQueryResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartReplicationManagerRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerRequestProto;
@@ -114,6 +116,7 @@ import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ProtobufUtils;
import java.io.Closeable;
import java.io.IOException;
@@ -123,6 +126,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Consumer;
+import java.util.UUID;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMCloseContainerResponseProto.Status.CONTAINER_ALREADY_CLOSED;
@@ -486,6 +490,18 @@ public final class
StorageContainerLocationProtocolClientSideTranslatorPB
return response.getDatanodesList();
}
+ @Override
+ public HddsProtos.Node queryNode(UUID uuid) throws IOException {
+ SingleNodeQueryRequestProto request =
SingleNodeQueryRequestProto.newBuilder()
+ .setUuid(ProtobufUtils.toProtobuf(uuid))
+ .build();
+ SingleNodeQueryResponseProto response =
+ submitRequest(Type.SingleNodeQuery,
+ builder -> builder.setSingleNodeQueryRequest(request))
+ .getSingleNodeQueryResponse();
+ return response.getDatanode();
+ }
+
/**
* Attempts to decommission the list of nodes.
* @param nodes The list of hostnames or hostname:ports to decommission
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 49e71d2fe6..6cfddcc2f6 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -78,9 +78,10 @@ message ScmContainerLocationRequest {
optional GetContainerReplicasRequestProto getContainerReplicasRequest = 39;
optional ReplicationManagerReportRequestProto
replicationManagerReportRequest = 40;
optional ResetDeletedBlockRetryCountRequestProto
resetDeletedBlockRetryCountRequest = 41;
- optional TransferLeadershipRequestProto transferScmLeadershipRequest = 42;
+ optional TransferLeadershipRequestProto transferScmLeadershipRequest = 42;
optional GetFailedDeletedBlocksTxnRequestProto
getFailedDeletedBlocksTxnRequest = 43;
optional DecommissionScmRequestProto decommissionScmRequest = 44;
+ optional SingleNodeQueryRequestProto singleNodeQueryRequest = 45;
}
message ScmContainerLocationResponse {
@@ -130,9 +131,10 @@ message ScmContainerLocationResponse {
optional GetContainerReplicasResponseProto getContainerReplicasResponse = 39;
optional ReplicationManagerReportResponseProto
getReplicationManagerReportResponse = 40;
optional ResetDeletedBlockRetryCountResponseProto
resetDeletedBlockRetryCountResponse = 41;
- optional TransferLeadershipResponseProto transferScmLeadershipResponse = 42;
+ optional TransferLeadershipResponseProto transferScmLeadershipResponse = 42;
optional GetFailedDeletedBlocksTxnResponseProto
getFailedDeletedBlocksTxnResponse = 43;
optional DecommissionScmResponseProto decommissionScmResponse = 44;
+ optional SingleNodeQueryResponseProto singleNodeQueryResponse = 45;
enum Status {
OK = 1;
@@ -184,6 +186,7 @@ enum Type {
TransferLeadership = 38;
GetFailedDeletedBlocksTransaction = 39;
DecommissionScm = 40;
+ SingleNodeQuery = 41;
}
/**
@@ -326,6 +329,14 @@ message NodeQueryResponseProto {
repeated Node datanodes = 1;
}
+message SingleNodeQueryRequestProto {
+ required UUID uuid = 1;
+}
+
+message SingleNodeQueryResponseProto {
+ optional Node datanode = 1;
+}
+
/*
Datanode usage info request message.
*/
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 7738d0e390..6d47a78a7d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -92,6 +92,8 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationRequest;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationResponse;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationResponse.Status;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SingleNodeQueryResponseProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SingleNodeQueryRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartMaintenanceNodesRequestProto;
@@ -120,6 +122,7 @@ import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
+import org.apache.hadoop.util.ProtobufUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -460,6 +463,13 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
.setNodeQueryResponse(queryNode(request.getNodeQueryRequest(),
request.getVersion()))
.build();
+ case SingleNodeQuery:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setSingleNodeQueryResponse(querySingleNode(request
+ .getSingleNodeQueryRequest()))
+ .build();
case CloseContainer:
return ScmContainerLocationResponse.newBuilder()
.setCmdType(request.getCmdType())
@@ -866,6 +876,16 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
.build();
}
+ public SingleNodeQueryResponseProto querySingleNode(
+ SingleNodeQueryRequestProto request)
+ throws IOException {
+
+ HddsProtos.Node datanode =
impl.queryNode(ProtobufUtils.fromProtobuf(request.getUuid()));
+ return SingleNodeQueryResponseProto.newBuilder()
+ .setDatanode(datanode)
+ .build();
+ }
+
public SCMCloseContainerResponseProto closeContainer(
SCMCloseContainerRequestProto request)
throws IOException {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 3d38fdbe81..ac92ea893d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -109,6 +109,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import java.util.UUID;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService.newReflectiveBlockingService;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
@@ -613,6 +614,27 @@ public class SCMClientProtocolServer implements
return result;
}
+ @Override
+ public HddsProtos.Node queryNode(UUID uuid)
+ throws IOException {
+ HddsProtos.Node result = null;
+ try {
+ DatanodeDetails node = scm.getScmNodeManager().getNodeByUuid(uuid);
+ if (node != null) {
+ NodeStatus ns = scm.getScmNodeManager().getNodeStatus(node);
+ result = HddsProtos.Node.newBuilder()
+ .setNodeID(node.getProtoBufMessage())
+ .addNodeStates(ns.getHealth())
+ .addNodeOperationalStates(ns.getOperationalState())
+ .build();
+ }
+ } catch (NodeNotFoundException e) {
+ throw new IOException(
+ "An unexpected error occurred querying the NodeStatus", e);
+ }
+ return result;
+ }
+
@Override
public List<DatanodeAdminError> decommissionNodes(List<String> nodes)
throws IOException {
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 7aa91cec73..1daffbb9b9 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -59,6 +59,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.UUID;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED_DEFAULT;
@@ -225,6 +226,11 @@ public class ContainerOperationClient implements ScmClient
{
queryScope, poolName, ClientVersion.CURRENT_VERSION);
}
+ @Override
+ public HddsProtos.Node queryNode(UUID uuid) throws IOException {
+ return storageContainerLocationClient.queryNode(uuid);
+ }
+
@Override
public List<DatanodeAdminError> decommissionNodes(List<String> hosts)
throws IOException {
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
index db12ee2aac..325e362d4f 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
@@ -29,6 +29,7 @@ import picocli.CommandLine;
import java.io.IOException;
import java.util.List;
+import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -82,6 +83,15 @@ public class ListInfoSubcommand extends ScmSubcommand {
@Override
public void execute(ScmClient scmClient) throws IOException {
pipelines = scmClient.listPipelines();
+ if (!Strings.isNullOrEmpty(uuid)) {
+ HddsProtos.Node node = scmClient.queryNode(UUID.fromString(uuid));
+ DatanodeWithAttributes dwa = new DatanodeWithAttributes(DatanodeDetails
+ .getFromProtoBuf(node.getNodeID()),
+ node.getNodeOperationalStates(0),
+ node.getNodeStates(0));
+ printDatanodeInfo(dwa);
+ return;
+ }
Stream<DatanodeWithAttributes> allNodes = getAllNodes(scmClient).stream();
if (!Strings.isNullOrEmpty(ipaddress)) {
allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress()
@@ -91,10 +101,6 @@ public class ListInfoSubcommand extends ScmSubcommand {
allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName()
.compareToIgnoreCase(hostname) == 0);
}
- if (!Strings.isNullOrEmpty(uuid)) {
- allNodes = allNodes.filter(p ->
- p.getDatanodeDetails().getUuidString().equals(uuid));
- }
if (!Strings.isNullOrEmpty(nodeOperationalState)) {
allNodes = allNodes.filter(p -> p.getOpState().toString()
.compareToIgnoreCase(nodeOperationalState) == 0);
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
index b6ae0a8ff4..1247b783b5 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import picocli.CommandLine;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -101,6 +102,32 @@ public class TestListInfoSubcommand {
assertTrue(m.find());
}
+ @Test
+ public void testDataNodeByUuidOutput()
+ throws Exception {
+ List<HddsProtos.Node> nodes = getNodeDetails();
+
+ ScmClient scmClient = mock(ScmClient.class);
+ when(scmClient.queryNode(any()))
+ .thenAnswer(invocation -> nodes.get(0));
+ when(scmClient.listPipelines())
+ .thenReturn(new ArrayList<>());
+
+ CommandLine c = new CommandLine(cmd);
+ c.parseArgs("--id", nodes.get(0).getNodeID().getUuid());
+ cmd.execute(scmClient);
+
+ Pattern p = Pattern.compile(
+ "^Operational State:\\s+IN_SERVICE$", Pattern.MULTILINE);
+ Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+
+ p = Pattern.compile(nodes.get(0).getNodeID().getUuid().toString(),
+ Pattern.MULTILINE);
+ m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+ }
+
private List<HddsProtos.Node> getNodeDetails() {
List<HddsProtos.Node> nodes = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
