This is an automated email from the ASF dual-hosted git repository.
ivandika3 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8f4595c082e HDDS-16193. Clean up ScmClient (#11035)
8f4595c082e is described below
commit 8f4595c082e6db74479e4736724d6124e7d7acac
Author: Eric C. Ho <[email protected]>
AuthorDate: Wed Aug 19 09:19:08 2026 +0800
HDDS-16193. Clean up ScmClient (#11035)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 67 +----------
.../hdds/scm/cli/ContainerOperationClient.java | 122 ---------------------
.../hadoop/hdds/scm/TestContainerOperations.java | 16 ++-
.../ozone/TestContainerBalancerOperations.java | 4 +-
4 files changed, 11 insertions(+), 198 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index cb4b8471a00..cd8bd619592 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -24,10 +24,10 @@
import java.util.Optional;
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.DeletedBlocksTransactionSummary;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoResponseProto;
@@ -53,15 +53,9 @@
* NOTE this is temporarily needed class. When SCM containers are full-fledged,
* this interface will likely be removed.
*/
[email protected]
@InterfaceStability.Unstable
public interface ScmClient extends Closeable {
- /**
- * Creates a Container on SCM and returns the pipeline.
- * @return ContainerInfo
- * @throws IOException
- */
- ContainerWithPipeline createContainer(String owner) throws IOException;
-
/**
* Gets a container by Name -- Throws if the container does not exist.
* @param containerId - Container ID
@@ -97,24 +91,6 @@ List<ContainerReplicaInfo> getContainerReplicas(
*/
void closeContainer(long containerId) throws IOException;
- /**
- * Deletes an existing container.
- * @param containerId - ID of the container.
- * @param pipeline - Pipeline that represents the container.
- * @param force - true to forcibly delete the container.
- * @throws IOException
- */
- void deleteContainer(long containerId, Pipeline pipeline, boolean force)
- throws IOException;
-
- /**
- * Deletes an existing container.
- * @param containerId - ID of the container.
- * @param force - true to forcibly delete the container.
- * @throws IOException
- */
- void deleteContainer(long containerId, boolean force) throws IOException;
-
/**
* Lists a range of containers and get their info.
*
@@ -164,45 +140,6 @@ ContainerListResult listContainer(long startContainerID,
int count,
Boolean suppressed)
throws IOException;
- /**
- * Read meta data from an existing container.
- * @param containerID - ID of the container.
- * @param pipeline - Pipeline where the container is located.
- * @return ContainerInfo
- * @throws IOException
- */
- ContainerDataProto readContainer(long containerID, Pipeline pipeline)
- throws IOException;
-
- /**
- * Read meta data from an existing container.
- * @param containerID - ID of the container.
- * @return ContainerInfo
- * @throws IOException
- */
- ContainerDataProto readContainer(long containerID)
- throws IOException;
-
- /**
- * Gets the container size -- Computed by SCM from Container Reports.
- * @param containerID - ID of the container.
- * @return number of bytes used by this container.
- * @throws IOException
- */
- long getContainerSize(long containerID) throws IOException;
-
- /**
- * Creates a Container on SCM and returns the pipeline.
- * @param type - Replication Type.
- * @param replicationFactor - Replication Factor
- * @return ContainerInfo
- * @throws IOException - in case of error.
- */
- @Deprecated
- ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
- HddsProtos.ReplicationFactor replicationFactor,
- String owner) throws IOException;
-
ContainerWithPipeline createContainer(ReplicationConfig replicationConfig,
String owner) throws IOException;
/**
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index c1973891d0a..4cb429269cd 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -19,10 +19,7 @@
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED_DEFAULT;
-import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
-import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import com.google.common.base.Preconditions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -30,13 +27,11 @@
import java.util.Optional;
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.SecretKeyProtocolScm;
-import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.DeletedBlocksTransactionSummary;
@@ -75,9 +70,6 @@ public class ContainerOperationClient implements ScmClient {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerOperationClient.class);
- private final long containerSizeB;
- private final HddsProtos.ReplicationFactor replicationFactor;
- private final HddsProtos.ReplicationType replicationType;
private final StorageContainerLocationProtocol
storageContainerLocationClient;
private final SecretKeyProtocolScm secretKeyClient;
@@ -106,18 +98,6 @@ public ContainerOperationClient(OzoneConfiguration conf,
ScmNodeTarget targetScm
storageContainerLocationClient = newContainerRpcClient(conf);
}
secretKeyClient = newSecretKeyClient(conf);
- containerSizeB = (int) conf.getStorageSize(OZONE_SCM_CONTAINER_SIZE,
- OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
- boolean useRatis = conf.getBoolean(
- ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY,
- ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_DEFAULT);
- if (useRatis) {
- replicationFactor = HddsProtos.ReplicationFactor.THREE;
- replicationType = HddsProtos.ReplicationType.RATIS;
- } else {
- replicationFactor = HddsProtos.ReplicationFactor.ONE;
- replicationType = HddsProtos.ReplicationType.STAND_ALONE;
- }
containerTokenEnabled = conf.getBoolean(HDDS_CONTAINER_TOKEN_ENABLED,
HDDS_CONTAINER_TOKEN_ENABLED_DEFAULT);
maxCountOfContainerList = conf
@@ -154,34 +134,6 @@ public static SecretKeyProtocolScm newSecretKeyClient(
return HddsServerUtil.getSecretKeyClientForSCM(configSource);
}
- @Override
- public ContainerWithPipeline createContainer(String owner)
- throws IOException {
- XceiverClientSpi client = null;
- XceiverClientManager clientManager = getXceiverClientManager();
- try {
- ContainerWithPipeline containerWithPipeline =
- storageContainerLocationClient.
- allocateContainer(replicationType, replicationFactor, owner);
-
- Pipeline pipeline = containerWithPipeline.getPipeline();
- client = clientManager.acquireClient(pipeline);
-
- Preconditions.checkState(
- pipeline.isOpen(),
- "Unexpected state=%s for pipeline=%s, expected state=%s",
- pipeline.getPipelineState(), pipeline.getId(),
- Pipeline.PipelineState.OPEN);
- createContainer(client,
- containerWithPipeline.getContainerInfo().getContainerID());
- return containerWithPipeline;
- } finally {
- if (client != null) {
- clientManager.releaseClient(client, false);
- }
- }
- }
-
/**
* Create a container over pipeline specified by the SCM.
*
@@ -212,14 +164,6 @@ public String getEncodedContainerToken(long containerId)
throws IOException {
.encodeToUrlString();
}
- @Override
- public ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
- HddsProtos.ReplicationFactor factor, String owner) throws IOException {
- ReplicationConfig replicationConfig =
- ReplicationConfig.fromProtoTypeAndFactor(replicationType, factor);
- return createContainer(replicationConfig, owner);
- }
-
@Override
public ContainerWithPipeline createContainer(ReplicationConfig
replicationConfig, String owner) throws IOException {
XceiverClientSpi client = null;
@@ -330,37 +274,6 @@ public void close() {
}
}
- @Override
- public void deleteContainer(long containerId, Pipeline pipeline,
- boolean force) throws IOException {
- XceiverClientSpi client = null;
- XceiverClientManager clientManager = getXceiverClientManager();
- try {
- String encodedToken = getEncodedContainerToken(containerId);
-
- client = clientManager.acquireClient(pipeline);
- ContainerProtocolCalls
- .deleteContainer(client, containerId, force, encodedToken);
- storageContainerLocationClient
- .deleteContainer(containerId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleted container {}, machines: {} ", containerId,
- pipeline.getNodes());
- }
- } finally {
- if (client != null) {
- clientManager.releaseClient(client, false);
- }
- }
- }
-
- @Override
- public void deleteContainer(long containerID, boolean force)
- throws IOException {
- ContainerWithPipeline info = getContainerWithPipeline(containerID);
- deleteContainer(containerID, info.getPipeline(), force);
- }
-
@Override
public ContainerListResult listContainer(long startContainerID,
int count) throws IOException {
@@ -405,28 +318,6 @@ public ContainerListResult listContainer(long
startContainerID,
startContainerID, count, state, repType, replicationConfig,
suppressed);
}
- @Override
- public ContainerDataProto readContainer(long containerID,
- Pipeline pipeline) throws IOException {
- XceiverClientManager clientManager = getXceiverClientManager();
- String encodedToken = getEncodedContainerToken(containerID);
- XceiverClientSpi client = null;
- try {
- client = clientManager.acquireClientForReadData(pipeline);
- ReadContainerResponseProto response = ContainerProtocolCalls
- .readContainer(client, containerID, encodedToken);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Read container {}, machines: {} ", containerID,
- pipeline.getNodes());
- }
- return response.getContainerData();
- } finally {
- if (client != null) {
- clientManager.releaseClient(client, false);
- }
- }
- }
-
public Map<DatanodeDetails, ReadContainerResponseProto>
readContainerFromAllNodes(long containerID, Pipeline pipeline)
throws IOException, InterruptedException {
XceiverClientManager clientManager = getXceiverClientManager();
@@ -445,12 +336,6 @@ public Map<DatanodeDetails, ReadContainerResponseProto>
readContainerFromAllNode
}
}
- @Override
- public ContainerDataProto readContainer(long containerID) throws IOException
{
- ContainerWithPipeline info = getContainerWithPipeline(containerID);
- return readContainer(containerID, info.getPipeline());
- }
-
@Override
public ContainerInfo getContainer(long containerId) throws
IOException {
@@ -484,13 +369,6 @@ public void closeContainer(long containerId)
storageContainerLocationClient.closeContainer(containerId);
}
- @Override
- public long getContainerSize(long containerID) throws IOException {
- // TODO : Fix this, it currently returns the capacity
- // but not the current usage.
- return containerSizeB;
- }
-
@Override
public boolean inSafeMode() throws IOException {
return storageContainerLocationClient.inSafeMode();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java
index 6740a1e0827..d8e0bf108d6 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -87,9 +88,8 @@ void cleanup() {
@Test
void testContainerStateMachineIdempotency() throws Exception {
- ContainerWithPipeline container = storageClient.createContainer(HddsProtos
- .ReplicationType.RATIS, HddsProtos.ReplicationFactor
- .ONE, OzoneConsts.OZONE);
+ ContainerWithPipeline container = storageClient.createContainer(
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
OzoneConsts.OZONE);
long containerID = container.getContainerInfo().getContainerID();
Pipeline pipeline = container.getPipeline();
XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
@@ -126,9 +126,8 @@ void testContainerStateMachineIdempotency() throws
Exception {
*/
@Test
public void testCreate() throws Exception {
- ContainerWithPipeline container = storageClient.createContainer(HddsProtos
- .ReplicationType.STAND_ALONE, HddsProtos.ReplicationFactor
- .ONE, OzoneConsts.OZONE);
+ ContainerWithPipeline container = storageClient.createContainer(
+
StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
OzoneConsts.OZONE);
assertEquals(container.getContainerInfo().getContainerID(), storageClient
.getContainer(container.getContainerInfo().getContainerID())
.getContainerID());
@@ -141,9 +140,8 @@ public void testCreate() throws Exception {
public void testListContainerExceedMaxAllowedCountOperations() throws
Exception {
// create n+1 containers
for (int i = 0; i < CONTAINER_LIST_LIMIT + 1; i++) {
- storageClient.createContainer(HddsProtos
- .ReplicationType.STAND_ALONE, HddsProtos.ReplicationFactor
- .ONE, OzoneConsts.OZONE);
+ storageClient.createContainer(
+
StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
OzoneConsts.OZONE);
}
int count = storageClient.listContainer(0, CONTAINER_LIST_LIMIT + 1)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
index 80d2c11079f..c4f76c2cfd6 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
@@ -31,6 +31,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
@@ -156,8 +157,7 @@ public void testIfCBCLIOverridesConfigs() throws Exception {
List<ContainerWithPipeline> createdContainers = new ArrayList<>(5);
for (int i = 0; i < 5; i++) {
createdContainers.add(containerBalancerClient.createContainer(
- HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.ONE,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
OzoneConsts.OZONE));
}
String excludedContainersList =
createdContainers.get(0).getContainerInfo().getContainerID() + ","
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
