This is an automated email from the ASF dual-hosted git repository.
chungen0126 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9eb3d1b1294 HDDS-16212. Avoid the per-group list copy in Recon
ContainerKeyMapperHelper (#11058)
9eb3d1b1294 is described below
commit 9eb3d1b1294ed4158030a204375d6169bb9e75e8
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Mon Aug 24 05:08:12 2026 +0800
HDDS-16212. Avoid the per-group list copy in Recon ContainerKeyMapperHelper
(#11058)
---
.../recon/tasks/ContainerKeyMapperHelper.java | 86 +++++++++++-----------
1 file changed, 45 insertions(+), 41 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperHelper.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperHelper.java
index 488cffae93b..9da69212f03 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperHelper.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperHelper.java
@@ -319,37 +319,39 @@ private static void handlePutOMKeyEvent(String key,
OmKeyInfo omKeyInfo,
long containerCountToIncrement = 0;
for (OmKeyLocationInfoGroup omKeyLocationInfoGroup :
omKeyInfo.getKeyLocationVersions()) {
long keyVersion = omKeyLocationInfoGroup.getVersion();
- for (OmKeyLocationInfo omKeyLocationInfo :
omKeyLocationInfoGroup.createLocationList()) {
- long containerId = omKeyLocationInfo.getContainerID();
- ContainerKeyPrefix containerKeyPrefix =
ContainerKeyPrefix.get(containerId, key, keyVersion);
- if
(reconContainerMetadataManager.getCountForContainerKeyPrefix(containerKeyPrefix)
== 0 &&
- !containerKeyMap.containsKey(containerKeyPrefix)) {
- // Save on writes. No need to save same container-key prefix
- // mapping again.
- containerKeyMap.put(containerKeyPrefix, 1);
- // Remove the container-key prefix from the deleted list if we
- // previously deleted it in this batch (and now we add it again)
- deletedContainerKeyList.remove(containerKeyPrefix);
-
- // check if container already exists and
- // increment the count of containers if it does not exist
- if (!reconContainerMetadataManager.doesContainerExists(containerId)
&&
- !containerKeyCountMap.containsKey(containerId)) {
- containerCountToIncrement++;
- }
+ for (List<OmKeyLocationInfo> locationList :
omKeyLocationInfoGroup.getLocationLists()) {
+ for (OmKeyLocationInfo omKeyLocationInfo : locationList) {
+ long containerId = omKeyLocationInfo.getContainerID();
+ ContainerKeyPrefix containerKeyPrefix =
ContainerKeyPrefix.get(containerId, key, keyVersion);
+ if
(reconContainerMetadataManager.getCountForContainerKeyPrefix(containerKeyPrefix)
== 0 &&
+ !containerKeyMap.containsKey(containerKeyPrefix)) {
+ // Save on writes. No need to save same container-key prefix
+ // mapping again.
+ containerKeyMap.put(containerKeyPrefix, 1);
+ // Remove the container-key prefix from the deleted list if we
+ // previously deleted it in this batch (and now we add it again)
+ deletedContainerKeyList.remove(containerKeyPrefix);
+
+ // check if container already exists and
+ // increment the count of containers if it does not exist
+ if
(!reconContainerMetadataManager.doesContainerExists(containerId) &&
+ !containerKeyCountMap.containsKey(containerId)) {
+ containerCountToIncrement++;
+ }
- // update the count of keys for the given containerID
- long keyCount;
- if (containerKeyCountMap.containsKey(containerId)) {
- keyCount = containerKeyCountMap.get(containerId);
- } else {
- keyCount =
reconContainerMetadataManager.getKeyCountForContainer(containerId);
- }
+ // update the count of keys for the given containerID
+ long keyCount;
+ if (containerKeyCountMap.containsKey(containerId)) {
+ keyCount = containerKeyCountMap.get(containerId);
+ } else {
+ keyCount =
reconContainerMetadataManager.getKeyCountForContainer(containerId);
+ }
- // increment the count and update containerKeyCount.
- // keyCount will be 0 if containerID is not found. So, there is no
- // need to initialize keyCount for the first time.
- containerKeyCountMap.put(containerId, ++keyCount);
+ // increment the count and update containerKeyCount.
+ // keyCount will be 0 if containerID is not found. So, there is no
+ // need to initialize keyCount for the first time.
+ containerKeyCountMap.put(containerId, ++keyCount);
+ }
}
}
}
@@ -490,18 +492,20 @@ public static void handleKeyReprocess(String key,
for (OmKeyLocationInfoGroup omKeyLocationInfoGroup :
omKeyInfo.getKeyLocationVersions()) {
long keyVersion = omKeyLocationInfoGroup.getVersion();
- for (OmKeyLocationInfo omKeyLocationInfo :
omKeyLocationInfoGroup.createLocationList()) {
- long containerId = omKeyLocationInfo.getContainerID();
- ContainerKeyPrefix containerKeyPrefix =
ContainerKeyPrefix.get(containerId, key, keyVersion);
-
- // During reprocess, tables are empty so skip DB lookup - just check
in-memory map
- if (!localContainerKeyMap.containsKey(containerKeyPrefix)) {
- // Save on writes. No need to save same container-key prefix mapping
again.
- localContainerKeyMap.put(containerKeyPrefix, 1);
-
- // Thread-safe increment using computeIfAbsent (cross-task safe: FSO
+ OBS)
- sharedContainerKeyCountMap.computeIfAbsent(containerId, k -> new
AtomicLong(0))
- .incrementAndGet();
+ for (List<OmKeyLocationInfo> locationList :
omKeyLocationInfoGroup.getLocationLists()) {
+ for (OmKeyLocationInfo omKeyLocationInfo : locationList) {
+ long containerId = omKeyLocationInfo.getContainerID();
+ ContainerKeyPrefix containerKeyPrefix =
ContainerKeyPrefix.get(containerId, key, keyVersion);
+
+ // During reprocess, tables are empty so skip DB lookup - just check
in-memory map
+ if (!localContainerKeyMap.containsKey(containerKeyPrefix)) {
+ // Save on writes. No need to save same container-key prefix
mapping again.
+ localContainerKeyMap.put(containerKeyPrefix, 1);
+
+ // Thread-safe increment using computeIfAbsent (cross-task safe:
FSO + OBS)
+ sharedContainerKeyCountMap.computeIfAbsent(containerId, k -> new
AtomicLong(0))
+ .incrementAndGet();
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
