This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new dd2e9c7175 HDDS-8577. [Snapshot] Disable compaction log when loading
metadata for snapshot (#4697)
dd2e9c7175 is described below
commit dd2e9c717579bf6749b6313bf6bdc04e8f7a1246
Author: Hemant Kumar <[email protected]>
AuthorDate: Wed May 17 00:28:06 2023 -0700
HDDS-8577. [Snapshot] Disable compaction log when loading metadata for
snapshot (#4697)
---
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 8 +++----
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 22 +++++++++++++----
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 28 +++++++++++++++++++---
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 9 +++----
.../hadoop/ozone/om/SstFilteringService.java | 2 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 6 ++++-
6 files changed, 58 insertions(+), 17 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
index 93d507de8e..bf420935bf 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
@@ -99,7 +99,7 @@ public final class DBStoreBuilder {
private boolean openReadOnly = false;
private int maxFSSnapshots = 0;
private final DBProfile defaultCfProfile;
- private boolean enableCompactionLog;
+ private boolean enableCompactionDag;
private boolean createCheckpointDirs = true;
// this is to track the total size of dbUpdates data since sequence
// number in request to avoid increase in heap memory.
@@ -210,7 +210,7 @@ public final class DBStoreBuilder {
return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs,
registry.build(), openReadOnly, maxFSSnapshots, dbJmxBeanNameName,
- enableCompactionLog, maxDbUpdatesSizeThreshold, createCheckpointDirs,
+ enableCompactionDag, maxDbUpdatesSizeThreshold, createCheckpointDirs,
configuration);
} finally {
tableConfigs.forEach(TableConfig::close);
@@ -268,8 +268,8 @@ public final class DBStoreBuilder {
return this;
}
- public DBStoreBuilder setEnableCompactionLog(boolean enableCompactionLog) {
- this.enableCompactionLog = enableCompactionLog;
+ public DBStoreBuilder setEnableCompactionDag(boolean enableCompactionDag) {
+ this.enableCompactionDag = enableCompactionDag;
return this;
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index b8bb40ee95..f5b71a5573 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -78,12 +78,13 @@ public class RDBStore implements DBStore {
// this is to track the total size of dbUpdates data since sequence
// number in request to avoid increase in heap memory.
private long maxDbUpdatesSizeThreshold;
+ private final ManagedDBOptions dbOptions;
@SuppressWarnings("parameternumber")
public RDBStore(File dbFile, ManagedDBOptions dbOptions,
ManagedWriteOptions writeOptions, Set<TableConfig> families,
CodecRegistry registry, boolean readOnly, int maxFSSnapshots,
- String dbJmxBeanNameName, boolean enableCompactionLog,
+ String dbJmxBeanNameName, boolean enableCompactionDag,
long maxDbUpdatesSizeThreshold,
boolean createCheckpointDirs,
ConfigurationSource configuration)
@@ -97,11 +98,12 @@ public class RDBStore implements DBStore {
dbLocation = dbFile;
dbJmxBeanName = dbJmxBeanNameName == null ? dbFile.getName() :
dbJmxBeanNameName;
+ this.dbOptions = dbOptions;
try {
- if (enableCompactionLog) {
+ if (enableCompactionDag) {
rocksDBCheckpointDiffer = RocksDBCheckpointDifferHolder.getInstance(
- dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR,
+ getSnapshotMetadataDir(),
DB_COMPACTION_SST_BACKUP_DIR, DB_COMPACTION_LOG_DIR,
dbLocation.toString(), configuration);
rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions);
@@ -140,7 +142,7 @@ public class RDBStore implements DBStore {
Files.createDirectories(snapshotsParentDirPath);
}
- if (enableCompactionLog) {
+ if (enableCompactionDag) {
ColumnFamily ssInfoTableCF = db.getColumnFamily(SNAPSHOT_INFO_TABLE);
Preconditions.checkNotNull(ssInfoTableCF,
"SnapshotInfoTable column family handle should not be null");
@@ -179,6 +181,10 @@ public class RDBStore implements DBStore {
}
}
+ public String getSnapshotMetadataDir() {
+ return dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR;
+ }
+
public String getSnapshotsParentDir() {
return snapshotsParentDir;
}
@@ -187,6 +193,14 @@ public class RDBStore implements DBStore {
return rocksDBCheckpointDiffer;
}
+
+ /**
+ * Returns the RocksDB's DBOptions.
+ */
+ public ManagedDBOptions getDbOptions() {
+ return dbOptions;
+ }
+
@Override
public void compactDB() throws IOException {
try (ManagedCompactRangeOptions options =
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index acd9237c2c..720407b854 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -82,6 +82,8 @@ import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMI
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertThrows;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -916,15 +918,35 @@ public class TestOmSnapshot {
public void testSnapshotOpensWithDisabledAutoCompaction() throws Exception {
String snapPrefix = createSnapshot(volumeName, bucketName);
RDBStore snapshotDBStore = (RDBStore)
- ((OmSnapshot)cluster.getOzoneManager().getOmSnapshotManager()
+ ((OmSnapshot)cluster.getOzoneManager().getOmSnapshotManager()
.checkForSnapshot(volumeName, bucketName, snapPrefix))
.getMetadataManager().getStore();
for (String table : snapshotDBStore.getTableNames().values()) {
Assertions.assertTrue(snapshotDBStore.getDb().getColumnFamily(table)
- .getHandle().getDescriptor()
- .getOptions().disableAutoCompactions());
+ .getHandle().getDescriptor()
+ .getOptions().disableAutoCompactions());
}
}
+ @Test
+ public void testCompactionDagDisableForSnapshotMetadata() throws Exception {
+ String snapshotName = createSnapshot(volumeName, bucketName);
+
+ RDBStore activeDbStore =
+ (RDBStore) cluster.getOzoneManager().getMetadataManager().getStore();
+ // RocksDBCheckpointDiffer should be not null for active DB store.
+ assertNotNull(activeDbStore.getRocksDBCheckpointDiffer());
+ assertEquals(2, activeDbStore.getDbOptions().listeners().size());
+
+ OmSnapshot omSnapshot = (OmSnapshot) cluster.getOzoneManager()
+ .getOmSnapshotManager()
+ .checkForSnapshot(volumeName, bucketName, snapshotName);
+
+ RDBStore snapshotDbStore =
+ (RDBStore) omSnapshot.getMetadataManager().getStore();
+ // RocksDBCheckpointDiffer should be null for snapshot DB store.
+ assertNull(snapshotDbStore.getRocksDBCheckpointDiffer());
+ assertEquals(0, snapshotDbStore.getDbOptions().listeners().size());
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 350164b3c2..2063afb72f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -388,7 +388,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpoint);
}
setStore(loadDB(conf, metaDir, dbName, false,
- java.util.Optional.of(Boolean.TRUE), false));
+ java.util.Optional.of(Boolean.TRUE), false, false));
initializeOmTables(false);
} catch (IOException e) {
stop();
@@ -530,7 +530,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir)
throws IOException {
return loadDB(configuration, metaDir, OM_DB_NAME, false,
- java.util.Optional.empty(), true);
+ java.util.Optional.empty(), true, true);
}
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
@@ -539,13 +539,14 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
disableAutoCompaction)
throws IOException {
return loadDB(configuration, metaDir, dbName, readOnly,
- disableAutoCompaction, true);
+ disableAutoCompaction, true, true);
}
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
String dbName, boolean readOnly,
java.util.Optional<Boolean>
disableAutoCompaction,
+ boolean enableCompactionDag,
boolean createCheckpointDirs)
throws IOException {
final int maxFSSnapshots = configuration.getInt(
@@ -557,7 +558,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
.setOpenReadOnly(readOnly)
.setPath(Paths.get(metaDir.getPath()))
.setMaxFSSnapshots(maxFSSnapshots)
- .setEnableCompactionLog(true)
+ .setEnableCompactionDag(enableCompactionDag)
.setCreateCheckpointDirs(createCheckpointDirs);
disableAutoCompaction.ifPresent(
dbStoreBuilder::disableDefaultCFAutoCompaction);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
index 5d3b988bc6..2b18886439 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
@@ -150,7 +150,7 @@ public class SstFilteringService extends BackgroundService {
try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl
.loadDB(ozoneManager.getConfiguration(),
new File(snapshotCheckpointDir),
- dbName, true, Optional.of(Boolean.TRUE), false)) {
+ dbName, true, Optional.of(Boolean.TRUE), false, false)) {
RocksDatabase db = rdbStore.getDb();
db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 9f588ba643..4719d06967 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -34,6 +34,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -55,6 +56,7 @@ import java.util.stream.Stream;
import org.apache.commons.io.file.PathUtils;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.utils.db.CodecRegistry;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool;
@@ -221,7 +223,9 @@ public class SnapshotDiffManager implements AutoCloseable {
.build()
);
- Path path = Paths.get(differ.getMetadataDir(), "snapDiff");
+ RDBStore rdbStore = (RDBStore)
ozoneManager.getMetadataManager().getStore();
+ Objects.requireNonNull(rdbStore, "DBStore can't be null.");
+ Path path = Paths.get(rdbStore.getSnapshotMetadataDir(), "snapDiff");
createEmptySnapDiffDir(path);
this.sstBackupDirForSnapDiffJobs = path.toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
