This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 880bb67d5cb HDDS-15077. Explain exclusiveSize and
exclusiveSizeDeltaFromDirDeepCleaning (#10544)
880bb67d5cb is described below
commit 880bb67d5cb6997b1a90d3c0e5137f3bc8eb41d1
Author: Wei-Chiu Chuang <[email protected]>
AuthorDate: Mon Jun 22 13:03:08 2026 -0700
HDDS-15077. Explain exclusiveSize and exclusiveSizeDeltaFromDirDeepCleaning
(#10544)
Generated-by: Gemini 3.5 Flash
---
hadoop-hdds/docs/content/feature/Snapshot.md | 18 ++++++
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 75 ++++++++++++++++++++--
2 files changed, 87 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdds/docs/content/feature/Snapshot.md
b/hadoop-hdds/docs/content/feature/Snapshot.md
index 3ac1d931d49..5a1cee340e7 100644
--- a/hadoop-hdds/docs/content/feature/Snapshot.md
+++ b/hadoop-hdds/docs/content/feature/Snapshot.md
@@ -48,6 +48,24 @@ When keys are changed or deleted in the live bucket, their
data blocks are retai
**Snapshot Data Storage:** Snapshot metadata resides in OM's RocksDB. Diff job
data is stored in `ozone.om.snapshot.diff.db.dir` (defaults to OM metadata
directory).
+### Snapshot Space & Size Tracking
+
+When a snapshot is created, it references the state of the bucket at that
point in time. Over time, as keys are deleted or overwritten in the active
namespace, the data blocks are kept alive by the snapshots. Ozone tracks space
usage for snapshots using the following metrics:
+
+#### Referenced Size
+* **`referencedSize`**: The total logical data size (in bytes, unreplicated)
of all active keys/files in the bucket at the moment the snapshot was created.
+* **`referencedReplicatedSize`**: The total replicated data size (in bytes,
replicated) referenced by the snapshot at its creation point.
+
+#### Exclusive Size
+As mutations occur in the active bucket or other snapshots, some blocks become
exclusively held by a single snapshot. Ozone tracks this exclusive size using
two separate asynchronous background services:
+
+1. **`KeyDeletingService` (Key Deep Cleaning)**: Processes deleted keys to
find blocks exclusively held by the snapshot. It sets `exclusiveSize`
(unreplicated) and `exclusiveReplicatedSize` (replicated).
+2. **`SnapshotDirectoryCleaningService` (Directory Deep Cleaning)**:
Recursively processes deleted directories. To avoid write conflicts and
overwriting between these two independent services, it stores its results
separately in `exclusiveSizeDeltaFromDirDeepCleaning` (unreplicated) and
`exclusiveReplicatedSizeDeltaFromDirDeepCleaning` (replicated).
+
+The actual total exclusive size of a snapshot is the sum of these fields:
+* **Total Exclusive Size**: `exclusiveSize` +
`exclusiveSizeDeltaFromDirDeepCleaning`
+* **Total Exclusive Replicated Size**: `exclusiveReplicatedSize` +
`exclusiveReplicatedSizeDeltaFromDirDeepCleaning`
+
For more details, see Prashant Pogde’s [Introducing Apache Ozone
Snapshots](https://medium.com/@prashantpogde/introducing-apache-ozone-snapshots-af82e976142f).
## User Tutorial
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
index 27b29871786..26e4c1beaec 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
@@ -72,11 +72,34 @@ public final class SnapshotInfo implements Auditable,
CopyObject<SnapshotInfo> {
private String snapshotPath; // snapshot mask
private boolean deepClean;
private boolean sstFiltered;
+ /**
+ * The total logical data size (in bytes, unreplicated) referenced by the
snapshot
+ * at the time of its creation.
+ */
private long referencedSize;
+ /**
+ * The total replicated data size (in bytes, replicated) referenced by the
snapshot
+ * at the time of its creation.
+ */
private long referencedReplicatedSize;
+ /**
+ * The amount of data (in bytes, unreplicated) exclusively referenced by
this snapshot,
+ * determined during key-level deep cleaning when KeyDeletingService
processes deleted keys.
+ */
private long exclusiveSize;
+ /**
+ * Same as exclusiveSize, but accounts for the replication factor.
+ */
private long exclusiveReplicatedSize;
+ /**
+ * The additional exclusive size (in bytes, unreplicated) discovered during
directory-level
+ * deep cleaning when SnapshotDirectoryCleaningService processes deleted
directories.
+ * Kept separate from exclusiveSize to avoid write overwrites between
asynchronous services.
+ */
private long exclusiveSizeDeltaFromDirDeepCleaning;
+ /**
+ * Same as exclusiveSizeDeltaFromDirDeepCleaning, but accounts for the
replication factor.
+ */
private long exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
private boolean deepCleanedDeletedDir;
private ByteString createTransactionInfo;
@@ -330,37 +353,53 @@ public Builder setSstFiltered(boolean sstFiltered) {
return this;
}
- /** @param referencedSize - Snapshot referenced size. */
+ /**
+ * @param referencedSize - The total logical data size (in bytes,
unreplicated)
+ * referenced by the snapshot at the time of its creation.
+ */
public Builder setReferencedSize(long referencedSize) {
this.referencedSize = referencedSize;
return this;
}
- /** @param referencedReplicatedSize - Snapshot referenced size w/
replication. */
+ /**
+ * @param referencedReplicatedSize - Same as referencedSize, but scaled by
the replication factor.
+ */
public Builder setReferencedReplicatedSize(long referencedReplicatedSize) {
this.referencedReplicatedSize = referencedReplicatedSize;
return this;
}
- /** @param exclusiveSize - Snapshot exclusive size. */
+ /**
+ * @param exclusiveSize - The amount of data (in bytes, unreplicated)
exclusively
+ * referenced by this snapshot, determined during key-level deep cleaning.
+ */
public Builder setExclusiveSize(long exclusiveSize) {
this.exclusiveSize = exclusiveSize;
return this;
}
- /** @param exclusiveReplicatedSize - Snapshot exclusive size w/
replication. */
+ /**
+ * @param exclusiveReplicatedSize - Same as exclusiveSize, but scaled by
the replication factor.
+ */
public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
return this;
}
- /** @param exclusiveSizeDeltaFromDirDeepCleaning - Snapshot exclusive
size. */
+ /**
+ * @param exclusiveSizeDeltaFromDirDeepCleaning - The additional exclusive
size (in bytes,
+ * unreplicated) discovered during directory-level deep cleaning.
+ */
public Builder setExclusiveSizeDeltaFromDirDeepCleaning(long
exclusiveSizeDeltaFromDirDeepCleaning) {
this.exclusiveSizeDeltaFromDirDeepCleaning =
exclusiveSizeDeltaFromDirDeepCleaning;
return this;
}
- /** @param exclusiveReplicatedSizeDeltaFromDirDeepCleaning - Snapshot
exclusive size w/ replication. */
+ /**
+ * @param exclusiveReplicatedSizeDeltaFromDirDeepCleaning - Same as
+ * exclusiveSizeDeltaFromDirDeepCleaning, but scaled by the replication
factor.
+ */
public Builder setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(
long exclusiveReplicatedSizeDeltaFromDirDeepCleaning) {
this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning =
exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
@@ -567,6 +606,10 @@ public void setReferencedSize(long referencedSize) {
this.referencedSize = referencedSize;
}
+ /**
+ * Returns the total logical data size (in bytes, unreplicated) referenced
by the snapshot
+ * at the time of its creation.
+ */
public long getReferencedSize() {
return referencedSize;
}
@@ -575,6 +618,10 @@ public void setReferencedReplicatedSize(long
referencedReplicatedSize) {
this.referencedReplicatedSize = referencedReplicatedSize;
}
+ /**
+ * Returns the total replicated data size (in bytes, replicated) referenced
by the snapshot
+ * at the time of its creation.
+ */
public long getReferencedReplicatedSize() {
return referencedReplicatedSize;
}
@@ -583,6 +630,10 @@ public void setExclusiveSize(long exclusiveSize) {
this.exclusiveSize = exclusiveSize;
}
+ /**
+ * Returns the unreplicated data size exclusively referenced by this
snapshot,
+ * calculated during key-level deep cleaning.
+ */
public long getExclusiveSize() {
return exclusiveSize;
}
@@ -591,6 +642,10 @@ public void setExclusiveSizeDeltaFromDirDeepCleaning(long
exclusiveSizeDeltaFrom
this.exclusiveSizeDeltaFromDirDeepCleaning =
exclusiveSizeDeltaFromDirDeepCleaning;
}
+ /**
+ * Returns the additional unreplicated data size discovered exclusively by
this snapshot
+ * during directory-level deep cleaning.
+ */
public long getExclusiveSizeDeltaFromDirDeepCleaning() {
return exclusiveSizeDeltaFromDirDeepCleaning;
}
@@ -603,10 +658,18 @@ public void
setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(long exclusiveRep
this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning =
exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
}
+ /**
+ * Returns the additional replicated data size discovered exclusively by
this snapshot
+ * during directory-level deep cleaning.
+ */
public long getExclusiveReplicatedSizeDeltaFromDirDeepCleaning() {
return exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
}
+ /**
+ * Returns the replicated data size exclusively referenced by this snapshot,
+ * calculated during key-level deep cleaning.
+ */
public long getExclusiveReplicatedSize() {
return exclusiveReplicatedSize;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
