This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c708259b56 HDDS-8215. Recon - SCM DB Insights. (#4569)
c708259b56 is described below
commit c708259b566526dc707ae3b80520a5e0c5defbb5
Author: devmadhuu <[email protected]>
AuthorDate: Thu May 4 14:33:56 2023 +0530
HDDS-8215. Recon - SCM DB Insights. (#4569)
---
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +
.../hadoop/ozone/recon/api/BlocksEndPoint.java | 174 +++++++++++++
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 118 +++++++--
.../api/types/ContainerBlocksInfoWrapper.java | 82 ++++++
.../recon/api/types/DeletedContainerInfo.java | 142 +++++++++++
.../scm/ReconStorageContainerManagerFacade.java | 6 +
.../hadoop/ozone/recon/api/TestBlocksEndPoint.java | 281 +++++++++++++++++++++
.../ozone/recon/api/TestContainerEndpoint.java | 242 +++++++++++++++---
8 files changed, 983 insertions(+), 64 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
index 5a013dc9c9..bea8478f30 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
@@ -49,6 +49,8 @@ public final class ReconConstants {
public static final String RECON_QUERY_BATCH_PARAM = "batchNum";
public static final String RECON_QUERY_PREVKEY = "prevKey";
public static final String PREV_CONTAINER_ID_DEFAULT_VALUE = "0";
+ public static final String PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE =
+ "0";
public static final String RECON_QUERY_LIMIT = "limit";
public static final String RECON_QUERY_VOLUME = "volume";
public static final String RECON_QUERY_BUCKET = "bucket";
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/BlocksEndPoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/BlocksEndPoint.java
new file mode 100644
index 0000000000..920d4122b0
--- /dev/null
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/BlocksEndPoint.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api;
+
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.recon.api.types.ContainerBlocksInfoWrapper;
+import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
+import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
+
+import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.DELETED_BLOCKS;
+import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
+import static
org.apache.hadoop.ozone.recon.ReconConstants.PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE;
+import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT;
+import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY;
+
+/**
+ * Endpoint to get following information about blocks metadata.
+ * Number of blocks pending deletion.
+ * - Blocks pending deletion for open/closing containers.
+ * - Blocks pending deletion for closed containers.
+ */
+@Path("/blocks")
+@Produces(MediaType.APPLICATION_JSON)
+@AdminOnly
+public class BlocksEndPoint {
+ private final DBStore scmDBStore;
+ private final ReconContainerManager containerManager;
+
+ @Inject
+ public BlocksEndPoint(ReconStorageContainerManagerFacade reconSCM) {
+ this.containerManager =
+ (ReconContainerManager) reconSCM.getContainerManager();
+ this.scmDBStore = reconSCM.getScmDBStore();
+ }
+
+ /**
+ * This API returns list of blocks grouped by container state
+ * (OPEN/CLOSING/CLOSED).
+ * {
+ * "OPEN": [
+ * {
+ * "containerId": 100,
+ * "localIDList": [
+ * 1,
+ * 2,
+ * 3,
+ * 4
+ * ],
+ * "localIDCount": 4,
+ * "txID": 1
+ * }
+ * ]
+ * }
+ * @param limit limits the number of records having list of blocks
+ * grouped by container state (OPEN/CLOSING/CLOSED)
+ * @param prevKey deletedBlocks table key to skip records before prevKey
+ * @return list of blocks grouped by container state (OPEN/CLOSING/CLOSED)
+ */
+ @GET
+ @Path("/deletePending")
+ public Response getBlocksPendingDeletion(
+ @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT)
+ int limit,
+ @DefaultValue(PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE)
+ @QueryParam(RECON_QUERY_PREVKEY) long prevKey) {
+ if (limit < 0 || prevKey < 0) {
+ // Send back an empty response
+ return Response.status(Response.Status.NOT_ACCEPTABLE).build();
+ }
+ Map<String, List<ContainerBlocksInfoWrapper>>
+ containerStateBlockInfoListMap = new HashMap<>();
+ try (
+ Table<Long,
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>
+ deletedBlocksTXTable = DELETED_BLOCKS.getTable(this.scmDBStore);
+ TableIterator<Long, ? extends Table.KeyValue<Long,
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>>
+ deletedBlocksTableIterator = deletedBlocksTXTable.iterator()) {
+ boolean skipPrevKey = false;
+ Long seekKey = prevKey;
+ if (prevKey > 0) {
+ skipPrevKey = true;
+ Table.KeyValue<Long,
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>
+ seekKeyValue =
+ deletedBlocksTableIterator.seek(seekKey);
+ // check if RocksDB was able to seek correctly to the given key prefix
+ // if not, then return empty result
+ if (seekKeyValue == null) {
+ return Response.ok(containerStateBlockInfoListMap).build();
+ }
+ }
+ while (deletedBlocksTableIterator.hasNext()) {
+ Table.KeyValue<Long,
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>
+ kv = deletedBlocksTableIterator.next();
+ Long key = kv.getKey();
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
+ deletedBlocksTransaction =
+ kv.getValue();
+ // skip the prev key if prev key is present
+ if (skipPrevKey && key.equals(prevKey)) {
+ continue;
+ }
+ long containerID = deletedBlocksTransaction.getContainerID();
+ String containerState =
+ containerManager.getContainer(ContainerID.valueOf(containerID))
+ .getState().name();
+ ContainerBlocksInfoWrapper containerBlocksInfoWrapper =
+ new ContainerBlocksInfoWrapper();
+ containerBlocksInfoWrapper.setContainerID(containerID);
+ containerBlocksInfoWrapper.setLocalIDList(
+ deletedBlocksTransaction.getLocalIDList());
+ containerBlocksInfoWrapper.setLocalIDCount(
+ deletedBlocksTransaction.getLocalIDCount());
+ containerBlocksInfoWrapper.setTxID(deletedBlocksTransaction.getTxID());
+ List<ContainerBlocksInfoWrapper> containerBlocksInfoWrappers;
+ if (containerStateBlockInfoListMap.containsKey(containerState)) {
+ containerBlocksInfoWrappers =
+ containerStateBlockInfoListMap.get(containerState);
+ } else {
+ containerBlocksInfoWrappers = new ArrayList<>();
+ containerStateBlockInfoListMap.put(containerState,
+ containerBlocksInfoWrappers);
+ }
+ containerBlocksInfoWrappers.add(containerBlocksInfoWrapper);
+ // limit is applied based on number of containers per state
+ if (containerBlocksInfoWrappers.size() >= limit) {
+ break;
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
+ } catch (Exception ex) {
+ throw new WebApplicationException(ex,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return Response.ok(containerStateBlockInfoListMap).build();
+ }
+}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
index de492f1ae9..5fe8a657cf 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
@@ -17,28 +17,8 @@
*/
package org.apache.hadoop.ozone.recon.api;
-import java.io.IOException;
-import java.time.Instant;
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import javax.inject.Inject;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
@@ -49,16 +29,17 @@ import
org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.DeletedContainerInfo;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
+import
org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
-import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
-import
org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
-import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
+import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager;
@@ -68,6 +49,26 @@ import
org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
import static
org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_BATCH_NUMBER;
import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
import static
org.apache.hadoop.ozone.recon.ReconConstants.PREV_CONTAINER_ID_DEFAULT_VALUE;
@@ -390,6 +391,72 @@ public class ContainerEndpoint {
return getUnhealthyContainers(null, limit, batchNum);
}
+ /**
+ * This API will return all DELETED containers in SCM in below JSON format.
+ * {
+ * containers: [
+ * {
+ * containerId: 1,
+ * state: DELETED,
+ * pipelineId: "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1",
+ * numOfKeys: 3,
+ * inStateSince: <stateEnterTime>
+ * },
+ * {
+ * containerId: 2,
+ * state: DELETED,
+ * pipelineId: "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1",
+ * numOfKeys: 6,
+ * inStateSince: <stateEnterTime>
+ * }
+ * ]
+ * }
+ * @param limit limits the number of deleted containers
+ * @param prevKey previous container Id to skip
+ * @return Response of delete containers.
+ */
+ @GET
+ @Path("/deleted")
+ public Response getSCMDeletedContainers(
+ @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT)
+ int limit,
+ @DefaultValue(PREV_CONTAINER_ID_DEFAULT_VALUE)
+ @QueryParam(RECON_QUERY_PREVKEY) long prevKey) {
+ List<DeletedContainerInfo> deletedContainerInfoList = new ArrayList<>();
+ try {
+ List<ContainerInfo> containers =
+ containerManager.getContainers(ContainerID.valueOf(prevKey), limit,
+ HddsProtos.LifeCycleState.DELETED);
+ containers = containers.stream()
+ .filter(containerInfo -> !(containerInfo.getContainerID() ==
prevKey))
+ .collect(
+ Collectors.toList());
+ containers.forEach(containerInfo -> {
+ DeletedContainerInfo deletedContainerInfo = new DeletedContainerInfo();
+ deletedContainerInfo.setContainerID(containerInfo.getContainerID());
+ deletedContainerInfo.setPipelineID(containerInfo.getPipelineID());
+ deletedContainerInfo.setNumberOfKeys(containerInfo.getNumberOfKeys());
+
deletedContainerInfo.setContainerState(containerInfo.getState().name());
+ deletedContainerInfo.setStateEnterTime(
+ containerInfo.getStateEnterTime().toEpochMilli());
+ deletedContainerInfo.setLastUsed(
+ containerInfo.getLastUsed().toEpochMilli());
+ deletedContainerInfo.setUsedBytes(containerInfo.getUsedBytes());
+ deletedContainerInfo.setReplicationConfig(
+ containerInfo.getReplicationConfig());
+ deletedContainerInfo.setReplicationFactor(
+ containerInfo.getReplicationFactor().name());
+ deletedContainerInfoList.add(deletedContainerInfo);
+ });
+ } catch (IllegalArgumentException e) {
+ throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
+ } catch (Exception ex) {
+ throw new WebApplicationException(ex,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return Response.ok(deletedContainerInfoList).build();
+ }
+
/**
* Helper function to extract the blocks for a given container from a given
* OM Key.
@@ -414,7 +481,4 @@ public class ContainerEndpoint {
return blockIds;
}
- private BucketLayout getBucketLayout() {
- return BucketLayout.DEFAULT;
- }
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerBlocksInfoWrapper.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerBlocksInfoWrapper.java
new file mode 100644
index 0000000000..35a270c225
--- /dev/null
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerBlocksInfoWrapper.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This class wraps containers and their associated blocks information.
+ */
+public class ContainerBlocksInfoWrapper {
+ @JsonProperty("containerId")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long containerID;
+ @JsonProperty("localIDList")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private List<Long> localIDList;
+ @JsonProperty("localIDCount")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private int localIDCount;
+ @JsonProperty("txID")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long txID;
+
+ public ContainerBlocksInfoWrapper() {
+ this.containerID = 0;
+ this.localIDList = new ArrayList<>();
+ this.localIDCount = 0;
+ this.txID = -1;
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public void setContainerID(long containerID) {
+ this.containerID = containerID;
+ }
+
+ public List<Long> getLocalIDList() {
+ return localIDList;
+ }
+
+ public void setLocalIDList(List<Long> localIDList) {
+ this.localIDList = localIDList;
+ }
+
+ public int getLocalIDCount() {
+ return localIDCount;
+ }
+
+ public void setLocalIDCount(int localIDCount) {
+ this.localIDCount = localIDCount;
+ }
+
+ public long getTxID() {
+ return txID;
+ }
+
+ public void setTxID(long txID) {
+ this.txID = txID;
+ }
+}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DeletedContainerInfo.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DeletedContainerInfo.java
new file mode 100644
index 0000000000..c1629dc950
--- /dev/null
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DeletedContainerInfo.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+
+/**
+ * This class wraps deleted container info for API response.
+ */
+public class DeletedContainerInfo {
+
+ @JsonProperty("containerId")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long containerID;
+
+ @JsonProperty("pipelineID")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private PipelineID pipelineID;
+
+ @JsonProperty("numberOfKeys")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long numberOfKeys;
+
+ @JsonProperty("containerState")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private String containerState;
+
+ @JsonProperty("stateEnterTime")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long stateEnterTime;
+
+ @JsonProperty("lastUsed")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long lastUsed;
+
+ @JsonProperty("usedBytes")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private long usedBytes;
+
+ @JsonProperty("replicationConfig")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private ReplicationConfig replicationConfig;
+
+ @JsonProperty("replicationFactor")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private String replicationFactor;
+
+ public DeletedContainerInfo() {
+
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public void setContainerID(long containerID) {
+ this.containerID = containerID;
+ }
+
+ public PipelineID getPipelineID() {
+ return pipelineID;
+ }
+
+ public void setPipelineID(PipelineID pipelineID) {
+ this.pipelineID = pipelineID;
+ }
+
+ public long getNumberOfKeys() {
+ return numberOfKeys;
+ }
+
+ public void setNumberOfKeys(long numberOfKeys) {
+ this.numberOfKeys = numberOfKeys;
+ }
+
+ public String getContainerState() {
+ return containerState;
+ }
+
+ public void setContainerState(String containerState) {
+ this.containerState = containerState;
+ }
+
+ public long getStateEnterTime() {
+ return stateEnterTime;
+ }
+
+ public void setStateEnterTime(long stateEnterTime) {
+ this.stateEnterTime = stateEnterTime;
+ }
+
+ public long getLastUsed() {
+ return lastUsed;
+ }
+
+ public void setLastUsed(long lastUsed) {
+ this.lastUsed = lastUsed;
+ }
+
+ public long getUsedBytes() {
+ return usedBytes;
+ }
+
+ public void setUsedBytes(long usedBytes) {
+ this.usedBytes = usedBytes;
+ }
+
+ public ReplicationConfig getReplicationConfig() {
+ return replicationConfig;
+ }
+
+ public void setReplicationConfig(
+ ReplicationConfig replicationConfig) {
+ this.replicationConfig = replicationConfig;
+ }
+
+ public String getReplicationFactor() {
+ return replicationFactor;
+ }
+
+ public void setReplicationFactor(String replicationFactor) {
+ this.replicationFactor = replicationFactor;
+ }
+}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index e7ab206cf4..22009392fa 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.inject.Singleton;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -118,6 +119,7 @@ import org.slf4j.LoggerFactory;
/**
* Recon's 'lite' version of SCM.
*/
+@Singleton
public class ReconStorageContainerManagerFacade
implements OzoneStorageContainerManager {
@@ -665,6 +667,10 @@ public class ReconStorageContainerManagerFacade
return reconNodeDetails;
}
+ public DBStore getScmDBStore() {
+ return dbStore;
+ }
+
public EventQueue getEventQueue() {
return eventQueue;
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestBlocksEndPoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestBlocksEndPoint.java
new file mode 100644
index 0000000000..385345a5e8
--- /dev/null
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestBlocksEndPoint.java
@@ -0,0 +1,281 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api;
+
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.recon.ReconTestInjector;
+import org.apache.hadoop.ozone.recon.api.types.ContainerBlocksInfoWrapper;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
+import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
+import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
+import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
+import
org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+import org.junit.rules.TemporaryFolder;
+
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static
org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.DELETED_BLOCKS;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Unit tests for APIs in BlocksEndPoint.
+ */
+public class TestBlocksEndPoint {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private ReconStorageContainerManagerFacade reconStorageContainerManager;
+ private ReconContainerManager reconContainerManager;
+ private ReconPipelineManager reconPipelineManager;
+ private BlocksEndPoint blocksEndPoint;
+ private boolean isSetupDone = false;
+ private ReconOMMetadataManager reconOMMetadataManager;
+ private DBStore scmDBStore;
+
+ private void initializeInjector() throws Exception {
+ reconOMMetadataManager = getTestReconOmMetadataManager(
+ initializeNewOmMetadataManager(temporaryFolder.newFolder()),
+ temporaryFolder.newFolder());
+
+ ReconTestInjector reconTestInjector =
+ new ReconTestInjector.Builder(temporaryFolder)
+ .withReconSqlDb()
+ .withReconOm(reconOMMetadataManager)
+ .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class))
+ // No longer using mock reconSCM as we need nodeDB in Facade
+ // to establish datanode UUID to hostname mapping
+ .addBinding(OzoneStorageContainerManager.class,
+ ReconStorageContainerManagerFacade.class)
+ .withContainerDB()
+ .addBinding(StorageContainerServiceProvider.class,
+ mock(StorageContainerServiceProviderImpl.class))
+ .addBinding(ContainerEndpoint.class)
+ .addBinding(BlocksEndPoint.class)
+ .build();
+
+ reconStorageContainerManager =
+
reconTestInjector.getInstance(ReconStorageContainerManagerFacade.class);
+ reconContainerManager = (ReconContainerManager)
+ reconStorageContainerManager.getContainerManager();
+ reconPipelineManager = (ReconPipelineManager)
+ reconStorageContainerManager.getPipelineManager();
+ blocksEndPoint = reconTestInjector.getInstance(
+ BlocksEndPoint.class);
+ scmDBStore = reconStorageContainerManager.getScmDBStore();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ // The following setup runs only once
+ if (!isSetupDone) {
+ initializeInjector();
+ isSetupDone = true;
+ }
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 100L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 101L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 102L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 103L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 104L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 105L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 106L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 107L));
+ }
+
+ @Test
+ public void testGetBlocksPendingDeletion() throws Exception {
+ List<Long> localIdList = new ArrayList<>();
+ localIdList.add(1L);
+ localIdList.add(2L);
+ localIdList.add(3L);
+ localIdList.add(4L);
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction dtx =
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
+ .newBuilder().setTxID(1).setContainerID(100L)
+ .addAllLocalID(localIdList).setCount(4).build();
+
+ Table<Long,
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>
+ deletedBlocksTable = DELETED_BLOCKS.getTable(this.scmDBStore);
+ deletedBlocksTable.put(1L, dtx);
+
+ Response blocksPendingDeletion =
+ blocksEndPoint.getBlocksPendingDeletion(1, 0);
+ Map<String, List<ContainerBlocksInfoWrapper>>
+ containerStateBlockInfoListMap =
+ (Map<String, List<ContainerBlocksInfoWrapper>>)
+ blocksPendingDeletion.getEntity();
+ Assertions.assertNotNull(containerStateBlockInfoListMap);
+ Assertions.assertEquals(1, containerStateBlockInfoListMap.size());
+ List<ContainerBlocksInfoWrapper> containerBlocksInfoWrappers =
+ containerStateBlockInfoListMap.get("OPEN");
+ ContainerBlocksInfoWrapper containerBlocksInfoWrapper =
+ containerBlocksInfoWrappers.get(0);
+ Assertions.assertEquals(100, containerBlocksInfoWrapper.getContainerID());
+ Assertions.assertEquals(4, containerBlocksInfoWrapper.getLocalIDCount());
+ Assertions.assertEquals(4,
+ containerBlocksInfoWrapper.getLocalIDList().size());
+ Assertions.assertEquals(1, containerBlocksInfoWrapper.getTxID());
+ }
+
+ @Test
+ public void testGetBlocksPendingDeletionLimitParam() throws Exception {
+ List<Long> localIdList = new ArrayList<>();
+ localIdList.add(1L);
+ localIdList.add(2L);
+ localIdList.add(3L);
+ localIdList.add(4L);
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction dtx =
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
+ .newBuilder().setTxID(1).setContainerID(100L)
+ .addAllLocalID(localIdList).setCount(4).build();
+
+ Table<Long,
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>
+ deletedBlocksTable = DELETED_BLOCKS.getTable(this.scmDBStore);
+ deletedBlocksTable.put(1L, dtx);
+
+ dtx =
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
+ .newBuilder().setTxID(2).setContainerID(101L)
+ .addAllLocalID(localIdList).setCount(4).build();
+ deletedBlocksTable.put(2L, dtx);
+
+ Response blocksPendingDeletion =
+ blocksEndPoint.getBlocksPendingDeletion(1, 0);
+ Map<String, List<ContainerBlocksInfoWrapper>>
+ containerStateBlockInfoListMap =
+ (Map<String, List<ContainerBlocksInfoWrapper>>)
+ blocksPendingDeletion.getEntity();
+ Assertions.assertNotNull(containerStateBlockInfoListMap);
+ Assertions.assertEquals(1, containerStateBlockInfoListMap.size());
+ List<ContainerBlocksInfoWrapper> containerBlocksInfoWrappers =
+ containerStateBlockInfoListMap.get("OPEN");
+ ContainerBlocksInfoWrapper containerBlocksInfoWrapper =
+ containerBlocksInfoWrappers.get(0);
+ Assertions.assertEquals(100, containerBlocksInfoWrapper.getContainerID());
+ Assertions.assertEquals(4, containerBlocksInfoWrapper.getLocalIDCount());
+ Assertions.assertEquals(4,
+ containerBlocksInfoWrapper.getLocalIDList().size());
+ Assertions.assertEquals(1, containerBlocksInfoWrapper.getTxID());
+ }
+
+ @Test
+ public void testGetBlocksPendingDeletionPrevKeyParam() throws Exception {
+ List<Long> localIdList = new ArrayList<>();
+ localIdList.add(1L);
+ localIdList.add(2L);
+ localIdList.add(3L);
+ localIdList.add(4L);
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction dtx =
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
+ .newBuilder().setTxID(1).setContainerID(100L)
+ .addAllLocalID(localIdList).setCount(4).build();
+
+ Table<Long,
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>
+ deletedBlocksTable = DELETED_BLOCKS.getTable(this.scmDBStore);
+ deletedBlocksTable.put(1L, dtx);
+
+ dtx =
+ StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
+ .newBuilder().setTxID(3).setContainerID(101L)
+ .addAllLocalID(localIdList).setCount(4).build();
+ deletedBlocksTable.put(3L, dtx);
+
+ Response blocksPendingDeletion =
+ blocksEndPoint.getBlocksPendingDeletion(1, 2);
+ Map<String, List<ContainerBlocksInfoWrapper>>
+ containerStateBlockInfoListMap =
+ (Map<String, List<ContainerBlocksInfoWrapper>>)
+ blocksPendingDeletion.getEntity();
+ Assertions.assertNotNull(containerStateBlockInfoListMap);
+ Assertions.assertEquals(1, containerStateBlockInfoListMap.size());
+ List<ContainerBlocksInfoWrapper> containerBlocksInfoWrappers =
+ containerStateBlockInfoListMap.get("OPEN");
+ ContainerBlocksInfoWrapper containerBlocksInfoWrapper =
+ containerBlocksInfoWrappers.get(0);
+ Assertions.assertEquals(101, containerBlocksInfoWrapper.getContainerID());
+ Assertions.assertEquals(4, containerBlocksInfoWrapper.getLocalIDCount());
+ Assertions.assertEquals(4,
+ containerBlocksInfoWrapper.getLocalIDList().size());
+ Assertions.assertEquals(3, containerBlocksInfoWrapper.getTxID());
+
+ blocksPendingDeletion =
+ blocksEndPoint.getBlocksPendingDeletion(1, 3);
+ containerStateBlockInfoListMap =
+ (Map<String, List<ContainerBlocksInfoWrapper>>)
+ blocksPendingDeletion.getEntity();
+ Assertions.assertTrue(containerStateBlockInfoListMap.size() == 0);
+
+ blocksPendingDeletion =
+ blocksEndPoint.getBlocksPendingDeletion(1, 4);
+ containerStateBlockInfoListMap =
+ (Map<String, List<ContainerBlocksInfoWrapper>>)
+ blocksPendingDeletion.getEntity();
+ Assertions.assertTrue(containerStateBlockInfoListMap.size() == 0);
+ }
+
+ protected ContainerWithPipeline getTestContainer(
+ HddsProtos.LifeCycleState state, long containerId)
+ throws IOException, TimeoutException {
+ ContainerID localContainerID = ContainerID.valueOf(containerId);
+ Pipeline localPipeline = getRandomPipeline();
+ reconPipelineManager.addPipeline(localPipeline);
+ ContainerInfo containerInfo =
+ new ContainerInfo.Builder()
+ .setContainerID(localContainerID.getId())
+ .setNumberOfKeys(10)
+ .setPipelineID(localPipeline.getId())
+ .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE))
+ .setOwner("test")
+ .setState(state)
+ .build();
+ return new ContainerWithPipeline(containerInfo, localPipeline);
+ }
+}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 2aa84e59ce..ee6f2c5444 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -18,66 +18,40 @@
package org.apache.hadoop.ozone.recon.api;
-import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getOmKeyLocationInfo;
-import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
-import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
-import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
-import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
-import static org.junit.Assert.assertNotNull;
-import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.recon.ReconTestInjector;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.DeletedContainerInfo;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
-import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
+import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
@@ -87,15 +61,46 @@ import
org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
import
org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
-import org.apache.hadoop.hdds.utils.db.Table;
import
org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
import org.junit.rules.TemporaryFolder;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getOmKeyLocationInfo;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
/**
* Test for container endpoint.
*/
@@ -103,9 +108,9 @@ public class TestContainerEndpoint {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
-
private OzoneStorageContainerManager ozoneStorageContainerManager;
private ReconContainerManager reconContainerManager;
+ private ContainerStateManager containerStateManager;
private ReconPipelineManager reconPipelineManager;
private ReconContainerMetadataManager reconContainerMetadataManager;
private ContainerEndpoint containerEndpoint;
@@ -182,6 +187,8 @@ public class TestContainerEndpoint {
pipeline = getRandomPipeline();
pipelineID = pipeline.getId();
reconPipelineManager.addPipeline(pipeline);
+ containerStateManager = reconContainerManager
+ .getContainerStateManager();
}
@Before
@@ -992,7 +999,168 @@ public class TestContainerEndpoint {
reconContainerManager.upsertContainerHistory(cID, uuid4, 4L, 1L);
}
- private BucketLayout getBucketLayout() {
- return BucketLayout.DEFAULT;
+ protected ContainerWithPipeline getTestContainer(
+ HddsProtos.LifeCycleState state, long containerId)
+ throws IOException, TimeoutException {
+ ContainerID localContainerID = ContainerID.valueOf(containerId);
+ Pipeline localPipeline = getRandomPipeline();
+ reconPipelineManager.addPipeline(localPipeline);
+ ContainerInfo containerInfo =
+ new ContainerInfo.Builder()
+ .setContainerID(localContainerID.getId())
+ .setNumberOfKeys(10)
+ .setPipelineID(localPipeline.getId())
+ .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE))
+ .setOwner("test")
+ .setState(state)
+ .build();
+ return new ContainerWithPipeline(containerInfo, localPipeline);
+ }
+
+ @Test
+ public void testGetSCMDeletedContainers() throws Exception {
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 102L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 103L));
+
+ reconContainerManager.updateContainerState(ContainerID.valueOf(102L),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ reconContainerManager.updateContainerState(ContainerID.valueOf(102L),
+ HddsProtos.LifeCycleEvent.CLOSE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(102L),
+ HddsProtos.LifeCycleEvent.DELETE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(102L),
+ HddsProtos.LifeCycleEvent.CLEANUP);
+ Set<ContainerID> containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETED);
+ Assertions.assertEquals(1, containerIDs.size());
+
+ reconContainerManager.updateContainerState(ContainerID.valueOf(103L),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ reconContainerManager.updateContainerState(ContainerID.valueOf(103L),
+ HddsProtos.LifeCycleEvent.CLOSE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(103L),
+ HddsProtos.LifeCycleEvent.DELETE);
+ containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETING);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(103L),
+ HddsProtos.LifeCycleEvent.CLEANUP);
+ containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETED);
+ Assertions.assertEquals(2, containerIDs.size());
+
+ Response scmDeletedContainers =
+ containerEndpoint.getSCMDeletedContainers(2, 0);
+ List<DeletedContainerInfo> deletedContainerInfoList =
+ (List<DeletedContainerInfo>) scmDeletedContainers.getEntity();
+ Assertions.assertEquals(2, deletedContainerInfoList.size());
+
+ DeletedContainerInfo deletedContainerInfo =
deletedContainerInfoList.get(0);
+ Assertions.assertEquals(102, deletedContainerInfo.getContainerID());
+ Assertions.assertEquals("DELETED",
+ deletedContainerInfo.getContainerState());
+
+ deletedContainerInfo = deletedContainerInfoList.get(1);
+ Assertions.assertEquals(103, deletedContainerInfo.getContainerID());
+ Assertions.assertEquals("DELETED",
+ deletedContainerInfo.getContainerState());
+ }
+
+ @Test
+ public void testGetSCMDeletedContainersLimitParam() throws Exception {
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 104L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 105L));
+ reconContainerManager.updateContainerState(ContainerID.valueOf(104L),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ reconContainerManager.updateContainerState(ContainerID.valueOf(104L),
+ HddsProtos.LifeCycleEvent.CLOSE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(104L),
+ HddsProtos.LifeCycleEvent.DELETE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(104L),
+ HddsProtos.LifeCycleEvent.CLEANUP);
+ Set<ContainerID> containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETED);
+ Assertions.assertEquals(1, containerIDs.size());
+
+ reconContainerManager.updateContainerState(ContainerID.valueOf(105L),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ reconContainerManager.updateContainerState(ContainerID.valueOf(105L),
+ HddsProtos.LifeCycleEvent.CLOSE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(105L),
+ HddsProtos.LifeCycleEvent.DELETE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(105L),
+ HddsProtos.LifeCycleEvent.CLEANUP);
+ containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETED);
+ Assertions.assertEquals(2, containerIDs.size());
+
+ Response scmDeletedContainers =
+ containerEndpoint.getSCMDeletedContainers(1, 0);
+ List<DeletedContainerInfo> deletedContainerInfoList =
+ (List<DeletedContainerInfo>) scmDeletedContainers.getEntity();
+ Assertions.assertEquals(1, deletedContainerInfoList.size());
+
+ DeletedContainerInfo deletedContainerInfo =
deletedContainerInfoList.get(0);
+ Assertions.assertEquals(104, deletedContainerInfo.getContainerID());
+ Assertions.assertEquals("DELETED",
+ deletedContainerInfo.getContainerState());
+ }
+
+ @Test
+ public void testGetSCMDeletedContainersPrevKeyParam() throws Exception {
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 106L));
+ reconContainerManager.addNewContainer(
+ getTestContainer(HddsProtos.LifeCycleState.OPEN, 107L));
+
+ reconContainerManager.updateContainerState(ContainerID.valueOf(106L),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ reconContainerManager.updateContainerState(ContainerID.valueOf(106L),
+ HddsProtos.LifeCycleEvent.CLOSE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(106L),
+ HddsProtos.LifeCycleEvent.DELETE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(106L),
+ HddsProtos.LifeCycleEvent.CLEANUP);
+ Set<ContainerID> containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETED);
+ Assertions.assertEquals(1, containerIDs.size());
+
+ reconContainerManager.updateContainerState(ContainerID.valueOf(107L),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ reconContainerManager.updateContainerState(ContainerID.valueOf(107L),
+ HddsProtos.LifeCycleEvent.CLOSE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(107L),
+ HddsProtos.LifeCycleEvent.DELETE);
+ reconContainerManager
+ .updateContainerState(ContainerID.valueOf(107L),
+ HddsProtos.LifeCycleEvent.CLEANUP);
+ containerIDs = containerStateManager
+ .getContainerIDs(HddsProtos.LifeCycleState.DELETED);
+ Assertions.assertEquals(2, containerIDs.size());
+
+ Response scmDeletedContainers =
+ containerEndpoint.getSCMDeletedContainers(2, 106L);
+ List<DeletedContainerInfo> deletedContainerInfoList =
+ (List<DeletedContainerInfo>) scmDeletedContainers.getEntity();
+ Assertions.assertEquals(1, deletedContainerInfoList.size());
+
+ DeletedContainerInfo deletedContainerInfo =
deletedContainerInfoList.get(0);
+ Assertions.assertEquals(107, deletedContainerInfo.getContainerID());
+ Assertions.assertEquals("DELETED",
+ deletedContainerInfo.getContainerState());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
