This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 50e6d616a3 HDDS-13022. Split up exclusive size tracking for key and
directory cleanup in SnapshotInfo (#8433)
50e6d616a3 is described below
commit 50e6d616a3222eb3289b5b0f11673c58596104a4
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Wed May 14 14:54:35 2025 -0400
HDDS-13022. Split up exclusive size tracking for key and directory cleanup
in SnapshotInfo (#8433)
---
.../apache/hadoop/ozone/client/OzoneSnapshot.java | 28 ++++-
.../hadoop/ozone/client/TestOzoneSnapshot.java | 63 +++++++++++
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 52 ++++++++-
.../ozone/om/helpers/TestOmSnapshotInfo.java | 9 +-
.../TestSnapshotDirectoryCleaningService.java | 9 +-
.../src/main/proto/OmClientProtocol.proto | 5 +
.../snapshot/OMSnapshotSetPropertyRequest.java | 116 ++++++++++++++-------
.../snapshot/OMSnapshotSetPropertyResponse.java | 15 +--
.../om/service/AbstractKeyDeletingService.java | 4 +
.../service/SnapshotDirectoryCleaningService.java | 4 +-
10 files changed, 249 insertions(+), 56 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
index 8297fdcc85..b7bf7051ca 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.client;
+import java.util.Objects;
import java.util.UUID;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
@@ -194,8 +195,31 @@ public static OzoneSnapshot fromSnapshotInfo(SnapshotInfo
snapshotInfo) {
snapshotInfo.getCheckpointDir(),
snapshotInfo.getReferencedSize(),
snapshotInfo.getReferencedReplicatedSize(),
- snapshotInfo.getExclusiveSize(),
- snapshotInfo.getExclusiveReplicatedSize()
+ snapshotInfo.getExclusiveSize() +
snapshotInfo.getExclusiveSizeDeltaFromDirDeepCleaning(),
+ snapshotInfo.getExclusiveReplicatedSize() +
snapshotInfo.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning()
);
}
+
+ @Override
+ public final boolean equals(Object o) {
+ if (!(o instanceof OzoneSnapshot)) {
+ return false;
+ }
+
+ OzoneSnapshot that = (OzoneSnapshot) o;
+ return creationTime == that.creationTime && referencedSize ==
that.referencedSize &&
+ referencedReplicatedSize == that.referencedReplicatedSize &&
exclusiveSize == that.exclusiveSize &&
+ exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
+ Objects.equals(volumeName, that.volumeName) &&
Objects.equals(bucketName, that.bucketName) &&
+ Objects.equals(name, that.name) && snapshotStatus ==
that.snapshotStatus &&
+ Objects.equals(snapshotId, that.snapshotId) &&
+ Objects.equals(snapshotPath, that.snapshotPath) &&
+ Objects.equals(checkpointDir, that.checkpointDir);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(volumeName, bucketName, name, creationTime,
snapshotStatus, snapshotId, snapshotPath,
+ checkpointDir, referencedSize, referencedReplicatedSize,
exclusiveSize, exclusiveReplicatedSize);
+ }
}
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneSnapshot.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneSnapshot.java
new file mode 100644
index 0000000000..8980e28b59
--- /dev/null
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneSnapshot.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.client;
+
+import static
org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.when;
+
+import java.util.UUID;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test class for OzoneSnapshot class.
+ */
+public class TestOzoneSnapshot {
+
+ private SnapshotInfo getMockedSnapshotInfo(UUID snapshotId) {
+ SnapshotInfo snapshotInfo = Mockito.mock(SnapshotInfo.class);
+ when(snapshotInfo.getVolumeName()).thenReturn("volume");
+ when(snapshotInfo.getBucketName()).thenReturn("bucket");
+ when(snapshotInfo.getName()).thenReturn("snap");
+ when(snapshotInfo.getCreationTime()).thenReturn(1000L);
+ when(snapshotInfo.getSnapshotStatus()).thenReturn(SNAPSHOT_ACTIVE);
+ when(snapshotInfo.getSnapshotId()).thenReturn(snapshotId);
+ when(snapshotInfo.getSnapshotPath()).thenReturn("volume/bucket");
+ when(snapshotInfo.getCheckpointDir()).thenReturn("checkpointDir");
+ when(snapshotInfo.getReferencedSize()).thenReturn(1000L);
+ when(snapshotInfo.getReferencedReplicatedSize()).thenReturn(3000L);
+ when(snapshotInfo.getExclusiveSize()).thenReturn(4000L);
+ when(snapshotInfo.getExclusiveReplicatedSize()).thenReturn(12000L);
+
when(snapshotInfo.getExclusiveSizeDeltaFromDirDeepCleaning()).thenReturn(2000L);
+
when(snapshotInfo.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning()).thenReturn(6000L);
+ return snapshotInfo;
+ }
+
+ @Test
+ public void testOzoneSnapshotFromSnapshotInfo() {
+ UUID snapshotId = UUID.randomUUID();
+ SnapshotInfo snapshotInfo = getMockedSnapshotInfo(snapshotId);
+ OzoneSnapshot ozoneSnapshot = OzoneSnapshot.fromSnapshotInfo(snapshotInfo);
+ OzoneSnapshot expectedOzoneSnapshot = new OzoneSnapshot(
+ "volume", "bucket", "snap", 1000L, SNAPSHOT_ACTIVE, snapshotId,
+ "volume/bucket", "checkpointDir", 1000L, 3000L, 6000L, 18000L);
+ assertEquals(expectedOzoneSnapshot, ozoneSnapshot);
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
index ab542a141b..4f4c9038f2 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
@@ -82,6 +82,8 @@ public final class SnapshotInfo implements Auditable,
CopyObject<SnapshotInfo> {
private long referencedReplicatedSize;
private long exclusiveSize;
private long exclusiveReplicatedSize;
+ private long exclusiveSizeDeltaFromDirDeepCleaning;
+ private long exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
private boolean deepCleanedDeletedDir;
private ByteString lastTransactionInfo;
@@ -104,6 +106,8 @@ private SnapshotInfo(Builder b) {
this.referencedReplicatedSize = b.referencedReplicatedSize;
this.exclusiveSize = b.exclusiveSize;
this.exclusiveReplicatedSize = b.exclusiveReplicatedSize;
+ this.exclusiveSizeDeltaFromDirDeepCleaning =
b.exclusiveSizeDeltaFromDirDeepCleaning;
+ this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning =
b.exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
this.deepCleanedDeletedDir = b.deepCleanedDeletedDir;
this.lastTransactionInfo = b.lastTransactionInfo;
}
@@ -233,6 +237,8 @@ public SnapshotInfo.Builder toBuilder() {
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize)
+
.setExclusiveSizeDeltaFromDirDeepCleaning(exclusiveSizeDeltaFromDirDeepCleaning)
+
.setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(exclusiveReplicatedSizeDeltaFromDirDeepCleaning)
.setDeepCleanedDeletedDir(deepCleanedDeletedDir)
.setLastTransactionInfo(lastTransactionInfo);
}
@@ -259,6 +265,8 @@ public static class Builder {
private long referencedReplicatedSize;
private long exclusiveSize;
private long exclusiveReplicatedSize;
+ private long exclusiveSizeDeltaFromDirDeepCleaning;
+ private long exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
private boolean deepCleanedDeletedDir;
private ByteString lastTransactionInfo;
@@ -374,6 +382,19 @@ public Builder setExclusiveReplicatedSize(long
exclusiveReplicatedSize) {
return this;
}
+ /** @param exclusiveSizeDeltaFromDirDeepCleaning - Snapshot exclusive
size. */
+ public Builder setExclusiveSizeDeltaFromDirDeepCleaning(long
exclusiveSizeDeltaFromDirDeepCleaning) {
+ this.exclusiveSizeDeltaFromDirDeepCleaning =
exclusiveSizeDeltaFromDirDeepCleaning;
+ return this;
+ }
+
+ /** @param exclusiveReplicatedSizeDeltaFromDirDeepCleaning - Snapshot
exclusive size w/ replication. */
+ public Builder setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(
+ long exclusiveReplicatedSizeDeltaFromDirDeepCleaning) {
+ this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning =
exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
+ return this;
+ }
+
public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
this.deepCleanedDeletedDir = deepCleanedDeletedDir;
return this;
@@ -408,6 +429,8 @@ public OzoneManagerProtocolProtos.SnapshotInfo
getProtobuf() {
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize)
+
.setExclusiveSizeDeltaFromDirDeepCleaning(exclusiveSizeDeltaFromDirDeepCleaning)
+
.setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(exclusiveReplicatedSizeDeltaFromDirDeepCleaning)
.setDeepCleanedDeletedDir(deepCleanedDeletedDir);
if (pathPreviousSnapshotId != null) {
@@ -485,6 +508,15 @@ public static SnapshotInfo getFromProtobuf(
snapshotInfoProto.getExclusiveReplicatedSize());
}
+ if (snapshotInfoProto.hasExclusiveSizeDeltaFromDirDeepCleaning()) {
+
osib.setExclusiveSizeDeltaFromDirDeepCleaning(snapshotInfoProto.getExclusiveSizeDeltaFromDirDeepCleaning());
+ }
+
+ if
(snapshotInfoProto.hasExclusiveReplicatedSizeDeltaFromDirDeepCleaning()) {
+ osib.setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(
+
snapshotInfoProto.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning());
+ }
+
if (snapshotInfoProto.hasDeepCleanedDeletedDir()) {
osib.setDeepCleanedDeletedDir(
snapshotInfoProto.getDeepCleanedDeletedDir());
@@ -571,10 +603,26 @@ public long getExclusiveSize() {
return exclusiveSize;
}
+ public void setExclusiveSizeDeltaFromDirDeepCleaning(long
exclusiveSizeDeltaFromDirDeepCleaning) {
+ this.exclusiveSizeDeltaFromDirDeepCleaning =
exclusiveSizeDeltaFromDirDeepCleaning;
+ }
+
+ public long getExclusiveSizeDeltaFromDirDeepCleaning() {
+ return exclusiveSizeDeltaFromDirDeepCleaning;
+ }
+
public void setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
}
+ public void setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(long
exclusiveReplicatedSizeDeltaFromDirDeepCleaning) {
+ this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning =
exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
+ }
+
+ public long getExclusiveReplicatedSizeDeltaFromDirDeepCleaning() {
+ return exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
+ }
+
public long getExclusiveReplicatedSize() {
return exclusiveReplicatedSize;
}
@@ -707,7 +755,9 @@ public String toString() {
", referencedReplicatedSize: '" + referencedReplicatedSize + '\'' +
", exclusiveSize: '" + exclusiveSize + '\'' +
", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' +
- ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
+ ", exclusiveSizeDeltaFromDirDeepCleaning: '" +
exclusiveSizeDeltaFromDirDeepCleaning + '\'' +
+ ", exclusiveReplicatedSizeDeltaFromDirDeepCleaning: '" +
exclusiveReplicatedSizeDeltaFromDirDeepCleaning +
+ "', deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
", lastTransactionInfo: '" + lastTransactionInfo + '\'' +
'}';
}
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
index 677458074a..eec64b90d4 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
@@ -67,7 +67,9 @@ private SnapshotInfo createSnapshotInfo() {
.setReferencedSize(2000L)
.setReferencedReplicatedSize(6000L)
.setExclusiveSize(1000L)
+ .setExclusiveSizeDeltaFromDirDeepCleaning(2000L)
.setExclusiveReplicatedSize(3000L)
+ .setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(6000L)
.setDeepCleanedDeletedDir(false)
.build();
}
@@ -92,6 +94,8 @@ private OzoneManagerProtocolProtos.SnapshotInfo
createSnapshotInfoProto() {
.setReferencedReplicatedSize(6000L)
.setExclusiveSize(1000L)
.setExclusiveReplicatedSize(3000L)
+ .setExclusiveSizeDeltaFromDirDeepCleaning(2000L)
+ .setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(6000L)
.setDeepCleanedDeletedDir(false)
.build();
}
@@ -179,7 +183,10 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
snapshotInfoActual.getExclusiveReplicatedSize());
assertEquals(snapshotInfoExpected.getDeepCleanedDeletedDir(),
snapshotInfoActual.getDeepCleanedDeletedDir());
-
+
assertEquals(snapshotInfoExpected.getExclusiveSizeDeltaFromDirDeepCleaning(),
+ snapshotInfoActual.getExclusiveSizeDeltaFromDirDeepCleaning());
+
assertEquals(snapshotInfoExpected.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning(),
+
snapshotInfoActual.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning());
assertEquals(snapshotInfoExpected, snapshotInfoActual);
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java
index 3f6e007ef4..84307c5549 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java
@@ -234,17 +234,18 @@ public void testExclusiveSizeWithDirectoryDeepClean()
throws Exception {
put("snap2", 5L);
put("snap3", 0L);
}};
- Thread.sleep(500);
try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>>
iterator = snapshotInfoTable.iterator()) {
while (iterator.hasNext()) {
Table.KeyValue<String, SnapshotInfo> snapshotEntry = iterator.next();
String snapshotName = snapshotEntry.getValue().getName();
- assertEquals(expectedSize.get(snapshotName), snapshotEntry.getValue().
- getExclusiveSize());
+ SnapshotInfo snapshotInfo =
snapshotInfoTable.get(snapshotEntry.getKey());
+ System.out.println(snapshotInfo.getName() + " " +
snapshotInfo.getDeepCleanedDeletedDir());
+ assertEquals(expectedSize.get(snapshotName),
+ snapshotInfo.getExclusiveSize() +
snapshotInfo.getExclusiveSizeDeltaFromDirDeepCleaning());
// Since for the test we are using RATIS/THREE
assertEquals(expectedSize.get(snapshotName) * 3,
- snapshotEntry.getValue().getExclusiveReplicatedSize());
+ snapshotInfo.getExclusiveReplicatedSize() +
snapshotInfo.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning());
}
}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index c7ff385016..b56912ff6d 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -303,6 +303,7 @@ message OMRequest {
optional GetObjectTaggingRequest getObjectTaggingRequest =
140;
optional PutObjectTaggingRequest putObjectTaggingRequest =
141;
optional DeleteObjectTaggingRequest deleteObjectTaggingRequest =
142;
+ repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests =
143;
}
message OMResponse {
@@ -892,6 +893,9 @@ message SnapshotInfo {
// note: shared sizes can be calculated from: referenced - exclusive
optional bool deepCleanedDeletedDir = 19;
optional bytes lastTransactionInfo = 20;
+ optional uint64 exclusiveSizeDeltaFromDirDeepCleaning = 21;
+ // snapshot exclusive size after replication
+ optional uint64 exclusiveReplicatedSizeDeltaFromDirDeepCleaning = 22;
}
message SnapshotDiffJobProto {
@@ -2029,6 +2033,7 @@ message SetSnapshotPropertyRequest {
optional SnapshotSize snapshotSize = 3;
optional bool deepCleanedDeletedDir = 4;
optional bool deepCleanedDeletedKey = 5;
+ optional SnapshotSize snapshotSizeDeltaFromDirDeepCleaning = 6;
}
// SnapshotProperty in entirely deprecated, Keeping it here for proto.lock
compatibility
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java
index 1fc847bf67..6491bd6391 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java
@@ -18,8 +18,16 @@
package org.apache.hadoop.ozone.om.request.snapshot;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+import com.google.common.collect.Lists;
import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -49,6 +57,32 @@ public OMSnapshotSetPropertyRequest(OMRequest omRequest) {
super(omRequest);
}
+ private void updateSnapshotProperty(
+ SnapshotInfo snapInfo,
OzoneManagerProtocolProtos.SetSnapshotPropertyRequest
setSnapshotPropertyRequest) {
+ if (setSnapshotPropertyRequest.hasDeepCleanedDeletedDir()) {
+ snapInfo.setDeepCleanedDeletedDir(setSnapshotPropertyRequest
+ .getDeepCleanedDeletedDir());
+ }
+
+ if (setSnapshotPropertyRequest.hasDeepCleanedDeletedKey()) {
+ snapInfo.setDeepClean(setSnapshotPropertyRequest
+ .getDeepCleanedDeletedKey());
+ }
+
+ if (setSnapshotPropertyRequest.hasSnapshotSize()) {
+ SnapshotSize snapshotSize = setSnapshotPropertyRequest.getSnapshotSize();
+ // Set Exclusive size.
+ snapInfo.setExclusiveSize(snapshotSize.getExclusiveSize());
+
snapInfo.setExclusiveReplicatedSize(snapshotSize.getExclusiveReplicatedSize());
+ }
+ if (setSnapshotPropertyRequest.hasSnapshotSizeDeltaFromDirDeepCleaning()) {
+ SnapshotSize snapshotSize =
setSnapshotPropertyRequest.getSnapshotSizeDeltaFromDirDeepCleaning();
+ // Set Exclusive size.
+
snapInfo.setExclusiveSizeDeltaFromDirDeepCleaning(snapshotSize.getExclusiveSize());
+
snapInfo.setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(snapshotSize.getExclusiveReplicatedSize());
+ }
+ }
+
@Override
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
ExecutionContext context) {
OMMetrics omMetrics = ozoneManager.getMetrics();
@@ -58,55 +92,57 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
OzoneManagerProtocolProtos.OMResponse.Builder omResponse =
OmResponseUtil.getOMResponseBuilder(getOmRequest());
- OzoneManagerProtocolProtos.SetSnapshotPropertyRequest
- setSnapshotPropertyRequest = getOmRequest()
- .getSetSnapshotPropertyRequest();
-
- String snapshotKey = setSnapshotPropertyRequest.getSnapshotKey();
-
+ List<OzoneManagerProtocolProtos.SetSnapshotPropertyRequest>
setSnapshotPropertyRequests = Lists.newArrayList();
+ if (getOmRequest().hasSetSnapshotPropertyRequest()) {
+
setSnapshotPropertyRequests.add(getOmRequest().getSetSnapshotPropertyRequest());
+ }
+
setSnapshotPropertyRequests.addAll(getOmRequest().getSetSnapshotPropertyRequestsList());
+ Set<String> snapshotKeys = new HashSet<>();
+ Map<String, SnapshotInfo> snapshotInfoMap = new HashMap<>();
try {
- SnapshotInfo updatedSnapInfo =
metadataManager.getSnapshotInfoTable().get(snapshotKey);
- if (updatedSnapInfo == null) {
- LOG.error("Snapshot: '{}' doesn't not exist in snapshot table.",
snapshotKey);
- throw new OMException("Snapshot: '{" + snapshotKey + "}' doesn't not
exist in snapshot table.", FILE_NOT_FOUND);
+ for (OzoneManagerProtocolProtos.SetSnapshotPropertyRequest
setSnapshotPropertyRequest :
+ setSnapshotPropertyRequests) {
+ String snapshotKey = setSnapshotPropertyRequest.getSnapshotKey();
+ if (snapshotKeys.contains(snapshotKey)) {
+ throw new OMException("Snapshot with snapshot key: " + snapshotKey +
" added multiple times in the request. "
+ + "Request: " + setSnapshotPropertyRequests, INVALID_REQUEST);
+ }
+ snapshotKeys.add(snapshotKey);
+ SnapshotInfo updatedSnapInfo =
snapshotInfoMap.computeIfAbsent(snapshotKey,
+ (k) -> {
+ try {
+ return metadataManager.getSnapshotInfoTable().get(k);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Exception while getting
key " + k, e);
+ }
+ });
+ if (updatedSnapInfo == null) {
+ LOG.error("Snapshot: '{}' doesn't not exist in snapshot table.",
snapshotKey);
+ throw new OMException("Snapshot: '{}' doesn't not exist in snapshot
table." + snapshotKey
+ + "Request: " + setSnapshotPropertyRequests, FILE_NOT_FOUND);
+ }
+ updateSnapshotProperty(updatedSnapInfo, setSnapshotPropertyRequest);
}
-
- if (setSnapshotPropertyRequest.hasDeepCleanedDeletedDir()) {
- updatedSnapInfo.setDeepCleanedDeletedDir(setSnapshotPropertyRequest
- .getDeepCleanedDeletedDir());
+ if (snapshotInfoMap.isEmpty()) {
+ throw new OMException("Snapshots: " + snapshotKeys + " don't not exist
in snapshot table.",
+ FILE_NOT_FOUND);
}
-
- if (setSnapshotPropertyRequest.hasDeepCleanedDeletedKey()) {
- updatedSnapInfo.setDeepClean(setSnapshotPropertyRequest
- .getDeepCleanedDeletedKey());
+ // Update Table Cache
+ for (Map.Entry<String, SnapshotInfo> snapshot :
snapshotInfoMap.entrySet()) {
+ metadataManager.getSnapshotInfoTable().addCacheEntry(
+ new CacheKey<>(snapshot.getKey()),
+ CacheValue.get(context.getIndex(), snapshot.getValue()));
+ omMetrics.incNumSnapshotSetProperties();
}
- if (setSnapshotPropertyRequest.hasSnapshotSize()) {
- SnapshotSize snapshotSize = setSnapshotPropertyRequest
- .getSnapshotSize();
- long exclusiveSize = updatedSnapInfo.getExclusiveSize() +
- snapshotSize.getExclusiveSize();
- long exclusiveReplicatedSize = updatedSnapInfo
- .getExclusiveReplicatedSize() + snapshotSize
- .getExclusiveReplicatedSize();
- // Set Exclusive size.
- updatedSnapInfo.setExclusiveSize(exclusiveSize);
- updatedSnapInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize);
- }
- // Update Table Cache
- metadataManager.getSnapshotInfoTable().addCacheEntry(
- new CacheKey<>(snapshotKey),
- CacheValue.get(context.getIndex(), updatedSnapInfo));
- omClientResponse = new OMSnapshotSetPropertyResponse(
- omResponse.build(), updatedSnapInfo);
- omMetrics.incNumSnapshotSetProperties();
- LOG.info("Successfully executed snapshotSetPropertyRequest: {{}}.",
setSnapshotPropertyRequest);
- } catch (IOException ex) {
+ omClientResponse = new OMSnapshotSetPropertyResponse(omResponse.build(),
snapshotInfoMap.values());
+ LOG.info("Successfully executed snapshotSetPropertyRequest: {{}}.",
setSnapshotPropertyRequests);
+ } catch (UncheckedIOException | IOException ex) {
omClientResponse = new OMSnapshotSetPropertyResponse(
createErrorOMResponse(omResponse, ex));
omMetrics.incNumSnapshotSetPropertyFails();
- LOG.error("Failed to execute snapshotSetPropertyRequest: {{}}.",
setSnapshotPropertyRequest, ex);
+ LOG.error("Failed to execute snapshotSetPropertyRequest: {{}}.",
setSnapshotPropertyRequests, ex);
}
return omClientResponse;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java
index 6f2bcddd11..5d0f8d642b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java
@@ -21,6 +21,7 @@
import jakarta.annotation.Nonnull;
import java.io.IOException;
+import java.util.Collection;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -33,26 +34,28 @@
*/
@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE})
public class OMSnapshotSetPropertyResponse extends OMClientResponse {
- private final SnapshotInfo updatedSnapInfo;
+ private final Collection<SnapshotInfo> updatedSnapInfos;
public OMSnapshotSetPropertyResponse(
@Nonnull OMResponse omResponse,
- @Nonnull SnapshotInfo updatedSnapInfo) {
+ @Nonnull Collection<SnapshotInfo> updatedSnapInfos) {
super(omResponse);
- this.updatedSnapInfo = updatedSnapInfo;
+ this.updatedSnapInfos = updatedSnapInfos;
}
public OMSnapshotSetPropertyResponse(@Nonnull OMResponse omResponse) {
super(omResponse);
checkStatusNotOK();
- this.updatedSnapInfo = null;
+ this.updatedSnapInfos = null;
}
@Override
protected void addToDBBatch(OMMetadataManager omMetadataManager,
BatchOperation batchOperation)
throws IOException {
- omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation,
- updatedSnapInfo.getTableKey(), updatedSnapInfo);
+ for (SnapshotInfo updatedSnapInfo : updatedSnapInfos) {
+ omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation,
+ updatedSnapInfo.getTableKey(), updatedSnapInfo);
+ }
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
index dc03b25a99..7dcb696e06 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
@@ -619,6 +619,10 @@ public AtomicLong getRunCount() {
return runCount;
}
+ public AtomicLong getCallId() {
+ return callId;
+ }
+
/**
* Returns the number of dirs deleted by the background service.
*
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
index ed79c13ff9..bb985da559 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
@@ -330,7 +330,7 @@ private void iterateDirectoryTree(
}
}
- private void updateExclusiveSize(String prevSnapshotKeyTable) {
+ private void updateExclusiveSize(String prevSnapshotKeyTable) throws
IOException {
ClientId clientId = ClientId.randomId();
SnapshotSize snapshotSize = SnapshotSize.newBuilder()
.setExclusiveSize(
@@ -345,7 +345,7 @@ private void updateExclusiveSize(String
prevSnapshotKeyTable) {
setSnapshotPropertyRequest =
SetSnapshotPropertyRequest.newBuilder()
.setSnapshotKey(prevSnapshotKeyTable)
- .setSnapshotSize(snapshotSize)
+ .setSnapshotSizeDeltaFromDirDeepCleaning(snapshotSize)
.build();
OMRequest omRequest = OMRequest.newBuilder()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
