This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 53c0a320a3 HDDS-12620. Fix OM Mismatch Deleted Container API (#8102)
53c0a320a3 is described below
commit 53c0a320a35fc487894f95ddd51d3df8e0eb1d4d
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Wed Mar 19 20:57:44 2025 -0700
HDDS-12620. Fix OM Mismatch Deleted Container API (#8102)
---
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 78 +++++++++++-----------
.../ozone/recon/api/TestContainerEndpoint.java | 36 ++++++++++
2 files changed, 74 insertions(+), 40 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
index b38e7138a1..af69fb88f9 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
@@ -675,9 +675,6 @@ public Response getContainerMisMatchInsights(
// Invalid filter parameter value
return Response.status(Response.Status.BAD_REQUEST).build();
}
- } catch (IOException ex) {
- throw new WebApplicationException(ex,
- Response.Status.INTERNAL_SERVER_ERROR);
} catch (IllegalArgumentException e) {
throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
} catch (Exception ex) {
@@ -725,43 +722,44 @@ public Response getOmContainersDeletedInSCM(
// Send back an empty response
return Response.status(Response.Status.NOT_ACCEPTABLE).build();
}
- List<ContainerDiscrepancyInfo> containerDiscrepancyInfoList =
- new ArrayList<>();
- try {
- Map<Long, ContainerMetadata> omContainers =
- reconContainerMetadataManager.getContainers(limit, prevKey);
-
- List<Long> deletedStateSCMContainerIds =
- containerManager.getContainers().stream()
- .filter(containerInfo -> (containerInfo.getState() ==
- HddsProtos.LifeCycleState.DELETED))
- .map(containerInfo -> containerInfo.getContainerID()).collect(
- Collectors.toList());
-
- List<Map.Entry<Long, ContainerMetadata>>
- omContainersDeletedInSCM =
- omContainers.entrySet().stream().filter(containerMetadataEntry ->
- (deletedStateSCMContainerIds.contains(
- containerMetadataEntry.getKey())))
- .collect(
- Collectors.toList());
-
- omContainersDeletedInSCM.forEach(
- containerMetadataEntry -> {
- ContainerDiscrepancyInfo containerDiscrepancyInfo =
- new ContainerDiscrepancyInfo();
- containerDiscrepancyInfo.setContainerID(
- containerMetadataEntry.getKey());
- containerDiscrepancyInfo.setNumberOfKeys(
- containerMetadataEntry.getValue().getNumberOfKeys());
- containerDiscrepancyInfo.setPipelines(
- containerMetadataEntry.getValue()
- .getPipelines());
- containerDiscrepancyInfoList.add(containerDiscrepancyInfo);
- });
- } catch (IOException ex) {
- throw new WebApplicationException(ex,
- Response.Status.INTERNAL_SERVER_ERROR);
+ if (limit <= 0) {
+ limit = Integer.MAX_VALUE;
+ }
+ long minContainerID = prevKey + 1;
+ Iterator<ContainerInfo> deletedStateSCMContainers =
containerManager.getContainers().stream()
+ .filter(containerInfo -> containerInfo.getContainerID() >=
minContainerID)
+ .filter(containerInfo -> containerInfo.getState() ==
HddsProtos.LifeCycleState.DELETED)
+
.sorted(Comparator.comparingLong(ContainerInfo::getContainerID)).iterator();
+ List<ContainerDiscrepancyInfo> containerDiscrepancyInfoList;
+ try (SeekableIterator<Long, ContainerMetadata> omContainers =
+ reconContainerMetadataManager.getContainersIterator()) {
+ ContainerInfo scmContainerInfo = deletedStateSCMContainers.hasNext() ?
deletedStateSCMContainers.next() : null;
+ ContainerMetadata containerMetadata = omContainers.hasNext() ?
omContainers.next() : null;
+ List<ContainerMetadata> omContainersDeletedInSCM = new ArrayList<>();
+ while (containerMetadata != null && scmContainerInfo != null
+ && omContainersDeletedInSCM.size() < limit) {
+ Long omContainerID = containerMetadata.getContainerID();
+ Long scmContainerID = scmContainerInfo.getContainerID();
+ if (scmContainerID.equals(omContainerID)) {
+ omContainersDeletedInSCM.add(containerMetadata);
+ scmContainerInfo = deletedStateSCMContainers.hasNext() ?
deletedStateSCMContainers.next() : null;
+ containerMetadata = omContainers.hasNext() ? omContainers.next() :
null;
+ } else if (scmContainerID.compareTo(omContainerID) < 0) {
+ scmContainerInfo = deletedStateSCMContainers.hasNext() ?
deletedStateSCMContainers.next() : null;
+ } else {
+ // Seek directly to scmContainerId iterating sequentially is very
wasteful here.
+ omContainers.seek(scmContainerID);
+ containerMetadata = omContainers.hasNext() ? omContainers.next() :
null;
+ }
+ }
+
+ containerDiscrepancyInfoList =
omContainersDeletedInSCM.stream().map(containerMetadataEntry -> {
+ ContainerDiscrepancyInfo containerDiscrepancyInfo = new
ContainerDiscrepancyInfo();
+
containerDiscrepancyInfo.setContainerID(containerMetadataEntry.getContainerID());
+
containerDiscrepancyInfo.setNumberOfKeys(containerMetadataEntry.getNumberOfKeys());
+
containerDiscrepancyInfo.setPipelines(containerMetadataEntry.getPipelines());
+ return containerDiscrepancyInfo;
+ }).collect(Collectors.toList());
} catch (IllegalArgumentException e) {
throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
} catch (Exception ex) {
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 65967c6e24..dd85f6c753 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -1583,6 +1583,42 @@ public void testGetOmContainersDeletedInSCM() throws
Exception {
assertEquals(1, containerDiscrepancyInfoList.size());
}
+ @Test
+ public void testGetOmContainersDeletedInSCMPagination() throws Exception {
+ Map<Long, ContainerMetadata> omContainers =
reconContainerMetadataManager.getContainers(-1, 0);
+ putContainerInfos(2);
+ List<ContainerInfo> scmContainers = reconContainerManager.getContainers();
+ assertEquals(omContainers.size(), scmContainers.size());
+ // Update container state of Container Id 2 to CLOSING to CLOSED
+ // and then to DELETED
+ updateContainerStateToDeleted(2);
+
+ assertContainerCount(HddsProtos.LifeCycleState.DELETED, 1);
+
+ List<ContainerInfo> deletedSCMContainers =
reconContainerManager.getContainers(HddsProtos.LifeCycleState.DELETED);
+ assertEquals(1, deletedSCMContainers.size());
+
+ Response omContainersDeletedInSCMResponse =
+ containerEndpoint.getOmContainersDeletedInSCM(1, 0);
+ assertNotNull(omContainersDeletedInSCMResponse);
+
+ Map<String, Object> responseMap = (Map<String, Object>)
omContainersDeletedInSCMResponse.getEntity();
+
+ // Fetch the ContainerDiscrepancyInfo list from the response
+ List<ContainerDiscrepancyInfo> containerDiscrepancyInfoList =
+ (List<ContainerDiscrepancyInfo>)
responseMap.get("containerDiscrepancyInfo");
+ assertEquals(1, containerDiscrepancyInfoList.size());
+ // Check the prevKey is set correct in the response
+ long responsePrevKey = (long) responseMap.get("lastKey");
+
assertEquals(containerDiscrepancyInfoList.get(containerDiscrepancyInfoList.size()
- 1).getContainerID(),
+ responsePrevKey);
+ assertEquals(2, responsePrevKey);
+
+ assertEquals(omContainers.get(2L).getNumberOfKeys(),
containerDiscrepancyInfoList.get(0).getNumberOfKeys());
+ assertEquals(1, containerDiscrepancyInfoList.size());
+ }
+
+
@Test
public void testGetOmContainersDeletedInSCMLimitParam() throws Exception {
Map<Long, ContainerMetadata> omContainers =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
