This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new eb5e25eaef HDDS-8784. trigger compaction outside of volume check.
(#6611)
eb5e25eaef is described below
commit eb5e25eaef5f1cac944a12552a87e1f24f7d1da7
Author: hao guo <[email protected]>
AuthorDate: Thu Aug 15 11:20:57 2024 +0800
HDDS-8784. trigger compaction outside of volume check. (#6611)
---
.../common/statemachine/DatanodeConfiguration.java | 43 ++++++++++++++++++++++
.../ozone/container/common/volume/HddsVolume.java | 26 ++++++++-----
.../container/common/volume/VolumeInfoMetrics.java | 8 ++++
.../ozone/container/ozoneimpl/OzoneContainer.java | 36 +++++++++++++++---
.../container/keyvalue/TestKeyValueContainer.java | 3 +-
5 files changed, 101 insertions(+), 15 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index a8b0d8cfa4..22dff7505c 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -110,6 +110,11 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
public static final Boolean
OZONE_DATANODE_CHECK_EMPTY_CONTAINER_DIR_ON_DELETE_DEFAULT = false;
+ private static final long
+ AUTO_COMPACTION_SMALL_SST_FILE_INTERVAL_MINUTES_DEFAULT = 120;
+ private static final int
+ AUTO_COMPACTION_SMALL_SST_FILE_THREADS_DEFAULT = 1;
+
/**
* Number of threads per volume that Datanode will use for chunk read.
*/
@@ -536,6 +541,24 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
)
private int autoCompactionSmallSstFileNum = 512;
+ @Config(key = "rocksdb.auto-compaction-small-sst-file.interval.minutes",
+ defaultValue = "120",
+ type = ConfigType.LONG,
+ tags = { DATANODE },
+ description = "Auto compact small SST files interval in minutes."
+ )
+ private long autoCompactionSmallSstFileIntervalMinutes =
+ AUTO_COMPACTION_SMALL_SST_FILE_INTERVAL_MINUTES_DEFAULT;
+
+ @Config(key = "rocksdb.auto-compaction-small-sst-file.threads",
+ defaultValue = "1",
+ type = ConfigType.INT,
+ tags = { DATANODE },
+ description = "Auto compact small SST files threads."
+ )
+ private int autoCompactionSmallSstFileThreads =
+ AUTO_COMPACTION_SMALL_SST_FILE_THREADS_DEFAULT;
+
/**
* Whether to check container directory or not to determine
* container is empty.
@@ -910,4 +933,24 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
public void setAutoCompactionSmallSstFileNum(int num) {
this.autoCompactionSmallSstFileNum = num;
}
+
+ public long getAutoCompactionSmallSstFileIntervalMinutes() {
+ return autoCompactionSmallSstFileIntervalMinutes;
+ }
+
+ public void setAutoCompactionSmallSstFileIntervalMinutes(
+ long autoCompactionSmallSstFileIntervalMinutes) {
+ this.autoCompactionSmallSstFileIntervalMinutes =
+ autoCompactionSmallSstFileIntervalMinutes;
+ }
+
+ public int getAutoCompactionSmallSstFileThreads() {
+ return autoCompactionSmallSstFileThreads;
+ }
+
+ public void setAutoCompactionSmallSstFileThreads(
+ int autoCompactionSmallSstFileThreads) {
+ this.autoCompactionSmallSstFileThreads =
+ autoCompactionSmallSstFileThreads;
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index d4cdaf2cfe..b22b9148bb 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.container.common.utils.RawDB;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import
org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures.SchemaV3;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -276,15 +277,6 @@ public class HddsVolume extends StorageVolume {
return VolumeCheckResult.FAILED;
}
- // TODO HDDS-8784 trigger compaction outside of volume check. Then the
- // exception can be removed.
- if (df.autoCompactionSmallSstFile()) {
- // Calculate number of files per level and size per level
- RawDB rawDB = DatanodeStoreCache.getInstance().getDB(
- dbFile.getAbsolutePath(), getConf());
- rawDB.getStore().compactionIfNeeded();
- }
-
return VolumeCheckResult.HEALTHY;
}
@@ -470,4 +462,20 @@ public class HddsVolume extends StorageVolume {
LOG.info("SchemaV3 db is stopped at {} for volume {}", containerDBPath,
getStorageID());
}
+
+ public void compactDb() {
+ File dbFile = new File(getDbParentDir(), CONTAINER_DB_NAME);
+ String dbFilePath = dbFile.getAbsolutePath();
+ try {
+ // Calculate number of files per level and size per level
+ RawDB rawDB =
+ DatanodeStoreCache.getInstance().getDB(dbFilePath, getConf());
+ long start = Time.monotonicNowNanos();
+ rawDB.getStore().compactionIfNeeded();
+ volumeInfoMetrics.dbCompactTimesNanoSecondsIncr(
+ Time.monotonicNowNanos() - start);
+ } catch (Exception e) {
+ LOG.warn("compact rocksdb error in {}", dbFilePath, e);
+ }
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
index e59cab0d53..68140600db 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
@@ -22,8 +22,10 @@ import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.ozone.OzoneConsts;
+
/**
* This class is used to track Volume Info stats for each HDDS Volume.
*/
@@ -33,6 +35,8 @@ public class VolumeInfoMetrics {
private String metricsSourceName = VolumeInfoMetrics.class.getSimpleName();
private final HddsVolume volume;
+ @Metric("Returns the RocksDB compact times of the Volume")
+ private MutableRate dbCompactLatency;
/**
* @param identifier Typically, path to volume root. E.g. /data/hdds
@@ -145,4 +149,8 @@ public class VolumeInfoMetrics {
return volume.getCommittedBytes();
}
+ public void dbCompactTimesNanoSecondsIncr(long time) {
+ dbCompactLatency.add(time);
+ }
+
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index b3809fff4b..5cdeaaa578 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -74,6 +74,9 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -122,7 +125,7 @@ public class OzoneContainer {
private final ReplicationServer replicationServer;
private DatanodeDetails datanodeDetails;
private StateContext context;
-
+ private ScheduledExecutorService dbCompactionExecutorService;
private final ContainerMetrics metrics;
@@ -158,9 +161,22 @@ public class OzoneContainer {
dbVolumeSet = HddsServerUtil.getDatanodeDbDirs(conf).isEmpty() ? null :
new MutableVolumeSet(datanodeDetails.getUuidString(), conf,
context, VolumeType.DB_VOLUME, volumeChecker);
+ final DatanodeConfiguration dnConf =
+ conf.getObject(DatanodeConfiguration.class);
if (SchemaV3.isFinalizedAndEnabled(config)) {
HddsVolumeUtil.loadAllHddsVolumeDbStore(
volumeSet, dbVolumeSet, false, LOG);
+ if (dnConf.autoCompactionSmallSstFile()) {
+ this.dbCompactionExecutorService = Executors.newScheduledThreadPool(
+ dnConf.getAutoCompactionSmallSstFileThreads(),
+ new ThreadFactoryBuilder().setNameFormat(
+ datanodeDetails.threadNamePrefix() +
+ "RocksDBCompactionThread-%d").build());
+
this.dbCompactionExecutorService.scheduleWithFixedDelay(this::compactDb,
+ dnConf.getAutoCompactionSmallSstFileIntervalMinutes(),
+ dnConf.getAutoCompactionSmallSstFileIntervalMinutes(),
+ TimeUnit.MINUTES);
+ }
}
long recoveringContainerTimeout = config.getTimeDuration(
@@ -221,8 +237,7 @@ public class OzoneContainer {
readChannel = new XceiverServerGrpc(
datanodeDetails, config, hddsDispatcher, certClient);
- Duration blockDeletingSvcInterval = conf.getObject(
- DatanodeConfiguration.class).getBlockDeletionInterval();
+ Duration blockDeletingSvcInterval = dnConf.getBlockDeletionInterval();
long blockDeletingServiceTimeout = config
.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
@@ -239,8 +254,8 @@ public class OzoneContainer {
datanodeDetails.threadNamePrefix(),
context.getParent().getReconfigurationHandler());
- Duration recoveringContainerScrubbingSvcInterval = conf.getObject(
- DatanodeConfiguration.class).getRecoveringContainerScrubInterval();
+ Duration recoveringContainerScrubbingSvcInterval =
+ dnConf.getRecoveringContainerScrubInterval();
long recoveringContainerScrubbingServiceTimeout = config
.getTimeDuration(OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT,
@@ -492,6 +507,9 @@ public class OzoneContainer {
if (dbVolumeSet != null) {
dbVolumeSet.shutdown();
}
+ if (dbCompactionExecutorService != null) {
+ dbCompactionExecutorService.shutdown();
+ }
blockDeletingService.shutdown();
recoveringContainerScrubbingService.shutdown();
ContainerMetrics.remove();
@@ -591,4 +609,12 @@ public class OzoneContainer {
return replicationServer;
}
+ public void compactDb() {
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ HddsVolume hddsVolume = (HddsVolume) volume;
+ CompletableFuture.runAsync(hddsVolume::compactDb,
+ dbCompactionExecutorService);
+ }
+ }
+
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 15d0faefdf..e1a3de30dd 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -137,6 +137,7 @@ public class TestKeyValueContainer {
CodecBuffer.enableLeakDetection();
DatanodeConfiguration dc = CONF.getObject(DatanodeConfiguration.class);
+ dc.setAutoCompactionSmallSstFile(true);
dc.setAutoCompactionSmallSstFileNum(100);
dc.setRocksdbDeleteObsoleteFilesPeriod(5000);
CONF.setFromObject(dc);
@@ -896,7 +897,7 @@ public class TestKeyValueContainer {
CONF).getStore();
List<LiveFileMetaData> fileMetaDataList1 =
((RDBStore)(dnStore.getStore())).getDb().getLiveFilesMetaData();
- hddsVolume.check(true);
+ hddsVolume.compactDb();
// Sleep a while to wait for compaction to complete
Thread.sleep(7000);
List<LiveFileMetaData> fileMetaDataList2 =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
