This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 89df7ed2722 HDDS-14037. Create DBDefinition and corresponding
MetadataManager for SnapshotDiff DB (#9398)
89df7ed2722 is described below
commit 89df7ed272249ca789c84b838b39c5f588082f6f
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Fri Dec 12 10:58:44 2025 -0500
HDDS-14037. Create DBDefinition and corresponding MetadataManager for
SnapshotDiff DB (#9398)
---
.../src/main/proto/OmStorageProtocol.proto | 5 +
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 47 +------
.../ozone/om/snapshot/SnapshotDiffManager.java | 13 +-
.../om/snapshot/db/SnapshotDiffDBDefinition.java | 145 +++++++++++++++++++
.../snapshot/db/SnapshotDiffMetadataManager.java | 51 +++++++
.../db/SnapshotDiffMetadataManagerImpl.java | 153 +++++++++++++++++++++
.../hadoop/ozone/om/snapshot/db/package-info.java | 21 +++
.../diff/helper/SnapshotDiffObjectInfo.java | 61 ++++++++
.../om/snapshot/diff/helper/package-info.java | 21 +++
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 4 +-
10 files changed, 470 insertions(+), 51 deletions(-)
diff --git
a/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto
b/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto
index ee57233f6de..2381e18d539 100644
--- a/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto
+++ b/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto
@@ -73,4 +73,9 @@ message FileSizeCountKeyProto {
required string volume = 1;
required string bucket = 2;
required int64 fileSizeUpperBound = 3;
+}
+
+message SnapDiffObjectInfo {
+ optional uint64 objectID = 1;
+ optional string keyName = 2;
}
\ No newline at end of file
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 2ca978dde29..77ed8a82487 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -53,6 +53,7 @@
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager.getSnapshotRootPath;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_PURGED_JOB_TABLE_NAME;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import com.google.common.annotations.VisibleForTesting;
@@ -103,6 +104,7 @@
import org.apache.hadoop.ozone.om.snapshot.SnapshotCache;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager;
import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
+import org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
@@ -132,43 +134,6 @@ public final class OmSnapshotManager implements
AutoCloseable {
public static final String DELIMITER = "-";
- /**
- * Contains all the snap diff job which are either queued, in_progress or
- * done. This table is used to make sure that there is only single job for
- * requests with the same snapshot pair at any point of time.
- * |------------------------------------------------|
- * | KEY | VALUE |
- * |------------------------------------------------|
- * | fromSnapshotId-toSnapshotId | SnapshotDiffJob |
- * |------------------------------------------------|
- */
- public static final String SNAP_DIFF_JOB_TABLE_NAME =
- "snap-diff-job-table";
-
- /**
- * Global table to keep the diff report. Each key is prefixed by the jobId
- * to improve look up and clean up. JobId comes from snap-diff-job-table.
- * |--------------------------------|
- * | KEY | VALUE |
- * |--------------------------------|
- * | jobId-index | DiffReportEntry |
- * |--------------------------------|
- */
- public static final String SNAP_DIFF_REPORT_TABLE_NAME =
- "snap-diff-report-table";
-
- /**
- * Contains all the snap diff job which can be purged either due to max
- * allowed time is over, FAILED or REJECTED.
- * |-------------------------------------------|
- * | KEY | VALUE |
- * |-------------------------------------------|
- * | jobId | numOfTotalEntriesInReportTable |
- * |-------------------------------------------|
- */
- private static final String SNAP_DIFF_PURGED_JOB_TABLE_NAME =
- "snap-diff-purged-job-table";
-
/**
* For snapshot compaction we need to capture SST files following column
* families before compaction.
@@ -248,9 +213,9 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws
IOException {
this.snapshotDiffDb = createRocksDbForSnapshotDiff(options,
dbPath, columnFamilyDescriptors, columnFamilyHandles);
- snapDiffJobCf = getOrCreateColumnFamily(SNAP_DIFF_JOB_TABLE_NAME,
+ snapDiffJobCf =
getOrCreateColumnFamily(SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_NAME,
columnFamilyDescriptors, columnFamilyHandles);
- snapDiffReportCf = getOrCreateColumnFamily(SNAP_DIFF_REPORT_TABLE_NAME,
+ snapDiffReportCf =
getOrCreateColumnFamily(SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME,
columnFamilyDescriptors, columnFamilyHandles);
snapDiffPurgedJobCf = getOrCreateColumnFamily(
SNAP_DIFF_PURGED_JOB_TABLE_NAME, columnFamilyDescriptors,
@@ -1090,8 +1055,8 @@ private void dropUnknownColumnFamilies(
List<ColumnFamilyHandle> familyHandles
) {
Set<String> allowedColumnFamilyOnStartUp = new HashSet<>(
- Arrays.asList(DEFAULT_COLUMN_FAMILY_NAME, SNAP_DIFF_JOB_TABLE_NAME,
- SNAP_DIFF_REPORT_TABLE_NAME, SNAP_DIFF_PURGED_JOB_TABLE_NAME));
+ Arrays.asList(DEFAULT_COLUMN_FAMILY_NAME,
SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_NAME,
+ SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME,
SNAP_DIFF_PURGED_JOB_TABLE_NAME));
try {
for (ColumnFamilyHandle columnFamilyHandle : familyHandles) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index ccb9e8c792d..4eeec3dc4c6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -116,6 +116,7 @@
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import org.apache.hadoop.ozone.om.helpers.WithParentObjectId;
+import org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition;
import
org.apache.hadoop.ozone.om.snapshot.diff.delta.CompositeDeltaDiffComputer;
import org.apache.hadoop.ozone.om.snapshot.diff.delta.DeltaFileComputer;
import org.apache.hadoop.ozone.om.snapshot.util.TableMergeIterator;
@@ -141,11 +142,7 @@
* Class to generate snapshot diff.
*/
public class SnapshotDiffManager implements AutoCloseable {
- private static final Logger LOG =
- LoggerFactory.getLogger(SnapshotDiffManager.class);
- private static final String FROM_SNAP_TABLE_SUFFIX = "-from-snap";
- private static final String TO_SNAP_TABLE_SUFFIX = "-to-snap";
- private static final String UNIQUE_IDS_TABLE_SUFFIX = "-unique-ids";
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotDiffManager.class);
private static final String DELETE_DIFF_TABLE_SUFFIX = "-delete-diff";
private static final String RENAME_DIFF_TABLE_SUFFIX = "-rename-diff";
private static final String CREATE_DIFF_TABLE_SUFFIX = "-create-diff";
@@ -816,11 +813,11 @@ void generateSnapshotDiffReport(final String jobKey,
// JobId is prepended to column families name to make them unique
// for request.
fromSnapshotColumnFamily =
- createColumnFamily(jobId + FROM_SNAP_TABLE_SUFFIX);
+ createColumnFamily(jobId +
SnapshotDiffDBDefinition.SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_NAME);
toSnapshotColumnFamily =
- createColumnFamily(jobId + TO_SNAP_TABLE_SUFFIX);
+ createColumnFamily(jobId +
SnapshotDiffDBDefinition.SNAP_DIFF_TO_SNAP_OBJECT_TABLE_NAME);
objectIDsColumnFamily =
- createColumnFamily(jobId + UNIQUE_IDS_TABLE_SUFFIX);
+ createColumnFamily(jobId +
SnapshotDiffDBDefinition.SNAP_DIFF_UNIQUE_IDS_TABLE_NAME);
// ObjectId to keyName map to keep key info for fromSnapshot.
// objectIdToKeyNameMap is used to identify what keys were touched
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java
new file mode 100644
index 00000000000..1add663c7f1
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.db;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DB_NAME;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR;
+import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone.getDiffReportEntryCodec;
+
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.BooleanCodec;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.hdds.utils.db.LongCodec;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
+import org.apache.hadoop.ozone.om.snapshot.diff.helper.SnapshotDiffObjectInfo;
+
+/**
+ * The SnapshotDiffDBDefinition class defines the schema for the snapshot
+ * difference database tables used in snapshot diff operations. Each table
+ * corresponds to a specific aspect of snapshot diff job processing and data
+ * storage, ensuring proper organization and efficient access.
+ */
+public final class SnapshotDiffDBDefinition extends DBDefinition.WithMap {
+ /**
+ * Contains all the snap diff job which are either queued, in_progress or
+ * done. This table is used to make sure that there is only single job for
+ * requests with the same snapshot pair at any point of time.
+ * |------------------------------------------------|
+ * | KEY | VALUE |
+ * |------------------------------------------------|
+ * | fromSnapshotId-toSnapshotId | SnapshotDiffJob |
+ * |------------------------------------------------|
+ */
+ public static final String SNAP_DIFF_JOB_TABLE_NAME = "snap-diff-job-table";
+
+ public static final DBColumnFamilyDefinition<String, SnapshotDiffJob>
SNAP_DIFF_JOB_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(SNAP_DIFF_JOB_TABLE_NAME,
StringCodec.get(), SnapshotDiffJob.getCodec());
+ /**
+ * Global table to keep the diff report. Each key is prefixed by the jobId
+ * to improve look up and clean up. JobId comes from snap-diff-job-table.
+ * |--------------------------------|
+ * | KEY | VALUE |
+ * |--------------------------------|
+ * | jobId-index | DiffReportEntry |
+ * |--------------------------------|
+ */
+ public static final String SNAP_DIFF_REPORT_TABLE_NAME =
"snap-diff-report-table";
+ public static final DBColumnFamilyDefinition<String, DiffReportEntry>
SNAP_DIFF_REPORT_TABLE_NAME_DEF
+ = new DBColumnFamilyDefinition<>(SNAP_DIFF_REPORT_TABLE_NAME,
StringCodec.get(), getDiffReportEntryCodec());
+ /**
+ * Contains all the snap diff job which can be purged either due to max
+ * allowed time is over, FAILED or REJECTED.
+ * |-------------------------------------------|
+ * | KEY | VALUE |
+ * |-------------------------------------------|
+ * | jobId | numOfTotalEntriesInReportTable |
+ * |-------------------------------------------|
+ */
+ public static final String SNAP_DIFF_PURGED_JOB_TABLE_NAME =
"snap-diff-purged-job-table";
+ public static final DBColumnFamilyDefinition<String, Long>
SNAP_DIFF_PURGED_JOB_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(SNAP_DIFF_PURGED_JOB_TABLE_NAME,
StringCodec.get(), LongCodec.get());
+ /**
+ * Contains all the snap diff job intermediate object output for from
snapshot.
+ * |----------------------------------------------------|
+ * | KEY | VALUE |
+ * |----------------------------------------------------|
+ * | jobId-objectId | SnapshotDiffObjectInfo |
+ * |----------------------------------------------------|
+ */
+ public static final String SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_NAME =
"-from-snap";
+ public static final DBColumnFamilyDefinition<String, SnapshotDiffObjectInfo>
SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_NAME,
StringCodec.get(),
+ SnapshotDiffObjectInfo.getCodec());
+
+ /**
+ * Contains all the snap diff job intermediate object output for to snapshot.
+ * |----------------------------------------------------|
+ * | KEY | VALUE |
+ * |----------------------------------------------------|
+ * | jobId-objectId | SnapshotDiffObjectInfo |
+ * |----------------------------------------------------|
+ */
+ public static final String SNAP_DIFF_TO_SNAP_OBJECT_TABLE_NAME = "-to-snap";
+ public static final DBColumnFamilyDefinition<String, SnapshotDiffObjectInfo>
SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(SNAP_DIFF_TO_SNAP_OBJECT_TABLE_NAME,
StringCodec.get(),
+ SnapshotDiffObjectInfo.getCodec());
+
+ /**
+ * Contains all the snap diff job intermediate object output for to snapshot.
+ * |----------------------------------------------------|
+ * | KEY | VALUE |
+ * |----------------------------------------------------|
+ * | jobId-objectId | boolean |
+ * |----------------------------------------------------|
+ */
+ public static final String SNAP_DIFF_UNIQUE_IDS_TABLE_NAME = "-unique-ids";
+ public static final DBColumnFamilyDefinition<String, Boolean>
SNAP_DIFF_UNIQUE_IDS_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(SNAP_DIFF_UNIQUE_IDS_TABLE_NAME,
StringCodec.get(), BooleanCodec.get());
+
+
//---------------------------------------------------------------------------//
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES =
+ DBColumnFamilyDefinition.newUnmodifiableMap(SNAP_DIFF_JOB_TABLE_DEF,
+ SNAP_DIFF_REPORT_TABLE_NAME_DEF,
+ SNAP_DIFF_PURGED_JOB_TABLE_DEF,
+ SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF,
+ SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF,
+ SNAP_DIFF_UNIQUE_IDS_TABLE_DEF);
+
+ private static final SnapshotDiffDBDefinition INSTANCE = new
SnapshotDiffDBDefinition();
+
+ public static SnapshotDiffDBDefinition get() {
+ return INSTANCE;
+ }
+
+ private SnapshotDiffDBDefinition() {
+ super(COLUMN_FAMILIES);
+ }
+
+ @Override
+ public String getName() {
+ return OM_SNAPSHOT_DIFF_DB_NAME;
+ }
+
+ @Override
+ public String getLocationConfigKey() {
+ return OZONE_OM_SNAPSHOT_DIFF_DB_DIR;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java
new file mode 100644
index 00000000000..667a49380c5
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.db;
+
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
+import org.apache.hadoop.ozone.om.snapshot.diff.helper.SnapshotDiffObjectInfo;
+
+/**
+ * Interface representing the metadata manager for snapshot difference
operations.
+ * This interface provides methods to access various tables storing metadata
+ * related to snapshot diff jobs, diff reports, purged jobs, and object
information
+ * for snapshots.
+ *
+ * Implementations of this interface should handle the setup, maintenance, and
+ * querying of the relevant metadata tables.
+ *
+ * The interface extends {@link AutoCloseable}, requiring implementations to
+ * handle proper resource cleanup.
+ */
+public interface SnapshotDiffMetadataManager extends AutoCloseable {
+
+ Table<String, SnapshotDiffJob> getSnapshotDiffJobTable();
+
+ Table<String, DiffReportEntry> getSnapshotDiffReportTable();
+
+ Table<String, Long> getSnapshotDiffPurgedJobTable();
+
+ Table<String, SnapshotDiffObjectInfo>
getSnapshotDiffFromSnapshotObjectInfoTable();
+
+ Table<String, SnapshotDiffObjectInfo>
getSnapshotDiffToSnapshotObjectInfoTable();
+
+ Table<String, Boolean> getSnapshotDiffUniqueObjectIdsTable();
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java
new file mode 100644
index 00000000000..68698160623
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.db;
+
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.getDBDirPath;
+import static
org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType.PARTIAL_CACHE;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_DEF;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_PURGED_JOB_TABLE_DEF;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME_DEF;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_UNIQUE_IDS_TABLE_DEF;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
+import org.apache.hadoop.ozone.om.snapshot.diff.helper.SnapshotDiffObjectInfo;
+
+/**
+ * Implementation of the {@link SnapshotDiffMetadataManager} interface.
+ * Provides functionality for managing metadata related to snapshot difference
+ * operations, including managing tables for snapshot diff jobs, reports,
purged jobs,
+ * and snapshot object information.
+ *
+ * This class handles the persistent storage of snapshot diff metadata using a
database.
+ * It ensures proper initialization and versioning of the metadata schema.
+ */
+public class SnapshotDiffMetadataManagerImpl implements
SnapshotDiffMetadataManager {
+
+ private static final String VERSION_FILE = "VERSION";
+ private static final String VERSION_NUMBER = "1";
+
+ private final DBStore dbStore;
+
+ private final Table<String, SnapshotDiffJob> snapshotDiffJobTable;
+ private final Table<String, SnapshotDiffReport.DiffReportEntry>
snapshotDiffReportTable;
+ private final Table<String, Long> snapshotDiffPurgedJobTable;
+ private final Table<String, SnapshotDiffObjectInfo>
snapshotDiffFromSnapshotObjectInfoTable;
+ private final Table<String, SnapshotDiffObjectInfo>
snapshotDiffToSnapshotObjectInfoTable;
+ private final Table<String, Boolean> snapshotDiffUniqueObjectIdsTable;
+
+ public SnapshotDiffMetadataManagerImpl(ConfigurationSource conf) throws
IOException {
+ File dbPath = getDBDirPath(SnapshotDiffDBDefinition.get(), conf).toPath()
+ .resolve(SnapshotDiffDBDefinition.get().getName()).toFile();
+ Path versionFilePath = dbPath.toPath().resolve(VERSION_FILE);
+ boolean versionContentMatches = checkAndDeleteIfVersionMismatches(dbPath,
versionFilePath);
+
+ this.dbStore = DBStoreBuilder.newBuilder(conf,
SnapshotDiffDBDefinition.get(), dbPath).build();
+
+ this.snapshotDiffJobTable = SNAP_DIFF_JOB_TABLE_DEF.getTable(dbStore,
PARTIAL_CACHE);
+ this.snapshotDiffReportTable =
SNAP_DIFF_REPORT_TABLE_NAME_DEF.getTable(dbStore, PARTIAL_CACHE);
+ this.snapshotDiffPurgedJobTable =
SNAP_DIFF_PURGED_JOB_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE);
+ this.snapshotDiffFromSnapshotObjectInfoTable =
+ SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE);
+ this.snapshotDiffToSnapshotObjectInfoTable =
SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE);
+ this.snapshotDiffUniqueObjectIdsTable =
SNAP_DIFF_UNIQUE_IDS_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE);
+ if (!versionContentMatches) {
+ writeVersionFileContent(versionFilePath);
+ }
+ }
+
+ /**
+ * Checks if the version specified in the version file matches the expected
version.
+ * If the versions do not match and the database path exists, the database
directory
+ * is deleted.
+ *
+ * @param dbPath the path to the database directory that may be deleted in
case of
+ * a version mismatch
+ * @param versionFilePath the path to the file containing the version
information
+ * @return {@code true} if the version matches, {@code false} otherwise
+ * @throws IOException if an I/O error occurs while reading the version file
or deleting
+ * the database directory
+ */
+ private boolean checkAndDeleteIfVersionMismatches(File dbPath, Path
versionFilePath) throws IOException {
+ boolean versionContentMatches =
VERSION_NUMBER.equals(getVersionFileContent(versionFilePath));
+ if (!versionContentMatches) {
+ if (dbPath.exists()) {
+ deleteDirectory(dbPath);
+ }
+ }
+ return versionContentMatches;
+ }
+
+ private static String getVersionFileContent(Path versionFilePath) throws
IOException {
+ if (versionFilePath.toFile().exists()) {
+ return
StringCodec.get().fromPersistedFormat(Files.readAllBytes(versionFilePath));
+ }
+ return null;
+ }
+
+ private static void writeVersionFileContent(Path versionFilePath) throws
IOException {
+ Files.write(versionFilePath,
StringCodec.get().toPersistedFormat(VERSION_NUMBER));
+ }
+
+ @Override
+ public Table<String, SnapshotDiffJob> getSnapshotDiffJobTable() {
+ return snapshotDiffJobTable;
+ }
+
+ @Override
+ public Table<String, SnapshotDiffReport.DiffReportEntry>
getSnapshotDiffReportTable() {
+ return snapshotDiffReportTable;
+ }
+
+ @Override
+ public Table<String, Long> getSnapshotDiffPurgedJobTable() {
+ return snapshotDiffPurgedJobTable;
+ }
+
+ @Override
+ public Table<String, SnapshotDiffObjectInfo>
getSnapshotDiffFromSnapshotObjectInfoTable() {
+ return snapshotDiffFromSnapshotObjectInfoTable;
+ }
+
+ @Override
+ public Table<String, SnapshotDiffObjectInfo>
getSnapshotDiffToSnapshotObjectInfoTable() {
+ return snapshotDiffToSnapshotObjectInfoTable;
+ }
+
+ @Override
+ public Table<String, Boolean> getSnapshotDiffUniqueObjectIdsTable() {
+ return snapshotDiffUniqueObjectIdsTable;
+ }
+
+ @Override
+ public void close() throws Exception {
+ dbStore.close();
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java
new file mode 100644
index 00000000000..3dc2ab8285f
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains classes for managing snapshot diff db.
+ */
+package org.apache.hadoop.ozone.om.snapshot.db;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java
new file mode 100644
index 00000000000..25c0f6d1526
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.helper;
+
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
+import
org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.SnapDiffObjectInfo;
+
+/**
+ * Represents information about an object in a snapshot difference.
+ * This class provides methods for converting between its internal
+ * representation and a Protobuf-based representation using a custom
+ * serialization codec.
+ */
+public class SnapshotDiffObjectInfo {
+
+ private static final Codec<SnapshotDiffObjectInfo> CODEC =
+ new
DelegatedCodec<>(Proto2Codec.get(SnapDiffObjectInfo.getDefaultInstance()),
+ SnapshotDiffObjectInfo::getFromProtobuf,
+ SnapshotDiffObjectInfo::toProtobuf,
+ SnapshotDiffObjectInfo.class);
+
+ private long objectId;
+ private String key;
+
+ public SnapshotDiffObjectInfo(long objectId, String key) {
+ this.objectId = objectId;
+ this.key = key;
+ }
+
+ public static Codec<SnapshotDiffObjectInfo> getCodec() {
+ return CODEC;
+ }
+
+ private static SnapshotDiffObjectInfo getFromProtobuf(SnapDiffObjectInfo
objectInfo) {
+ return new SnapshotDiffObjectInfo(objectInfo.getObjectID(),
objectInfo.getKeyName());
+ }
+
+ private SnapDiffObjectInfo toProtobuf() {
+ return SnapDiffObjectInfo.newBuilder()
+ .setObjectID(objectId)
+ .setKeyName(key)
+ .build();
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java
new file mode 100644
index 00000000000..7a96cecf814
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Helper classes for computing the delta between two snapshots.
+ */
+package org.apache.hadoop.ozone.om.snapshot.diff.helper;
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index 6ff6f235a63..5d681640249 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -30,13 +30,13 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
-import static
org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_JOB_TABLE_NAME;
-import static
org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_REPORT_TABLE_NAME;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY;
import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.getTableKey;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_NAME;
+import static
org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME;
import static
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_CANCELLED_JOB;
import static
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_DONE_JOB;
import static
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_FAILED_JOB;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
