This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 98df593 Allow force compact entry log when disabling entry log
compaction
98df593 is described below
commit 98df593ba432eb08b18d98331201099cc72879dc
Author: Yong Zhang <[email protected]>
AuthorDate: Mon Mar 8 19:04:18 2021 +0800
Allow force compact entry log when disabling entry log compaction
---
Master Issue: #2596
*Motivation*
Sometimes user will disable the entry log compaction to reduce the
I/O load. But we should allow forcing compact the entry even if
the entry log compaction is disabled.
*Modifications*
- Add a configuration to allow force compaction
Reviewers: Enrico Olivelli <[email protected]>, Jia Zhai
<[email protected]>, lipenghui <[email protected]>
This closes #2626 from zymap/allow-force-gc-without-interval
---
.../bookkeeper/bookie/GarbageCollectorThread.java | 16 +++++++--
.../bookkeeper/conf/ServerConfiguration.java | 22 +++++++++++++
.../apache/bookkeeper/bookie/CompactionTest.java | 38 ++++++++++++++++++++--
conf/bk_server.conf | 8 ++++-
4 files changed, 79 insertions(+), 5 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index b5d22d8..c242778 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -67,11 +67,13 @@ public class GarbageCollectorThread extends SafeRunnable {
final long gcWaitTime;
// Compaction parameters
+ boolean isForceMinorCompactionAllow = false;
boolean enableMinorCompaction = false;
final double minorCompactionThreshold;
final long minorCompactionInterval;
long lastMinorCompactionTime;
+ boolean isForceMajorCompactionAllow = false;
boolean enableMajorCompaction = false;
final double majorCompactionThreshold;
final long majorCompactionInterval;
@@ -178,6 +180,7 @@ public class GarbageCollectorThread extends SafeRunnable {
majorCompactionThreshold = conf.getMajorCompactionThreshold();
majorCompactionInterval = conf.getMajorCompactionInterval() * SECOND;
isForceGCAllowWhenNoSpace = conf.getIsForceGCAllowWhenNoSpace();
+ boolean isForceAllowCompaction = conf.isForceAllowCompaction();
AbstractLogCompactor.LogRemovalListener remover = new
AbstractLogCompactor.LogRemovalListener() {
@Override
@@ -203,6 +206,15 @@ public class GarbageCollectorThread extends SafeRunnable {
enableMinorCompaction = true;
}
+ if (isForceAllowCompaction) {
+ if (minorCompactionThreshold > 0 || minorCompactionThreshold <
1.0f) {
+ isForceMinorCompactionAllow = true;
+ }
+ if (majorCompactionThreshold > 0 || majorCompactionThreshold <
1.0f) {
+ isForceMajorCompactionAllow = true;
+ }
+ }
+
if (majorCompactionInterval > 0 && majorCompactionThreshold > 0) {
if (majorCompactionThreshold > 1.0f) {
throw new IOException("Invalid major compaction threshold "
@@ -344,7 +356,7 @@ public class GarbageCollectorThread extends SafeRunnable {
}
long curTime = System.currentTimeMillis();
- if (enableMajorCompaction && (!suspendMajor)
+ if ((isForceMajorCompactionAllow || enableMajorCompaction) &&
(!suspendMajor)
&& (force || curTime - lastMajorCompactionTime >
majorCompactionInterval)) {
// enter major compaction
LOG.info("Enter major compaction, suspendMajor {}", suspendMajor);
@@ -355,7 +367,7 @@ public class GarbageCollectorThread extends SafeRunnable {
lastMinorCompactionTime = lastMajorCompactionTime;
gcStats.getMajorCompactionCounter().inc();
majorCompacting.set(false);
- } else if (enableMinorCompaction && (!suspendMinor)
+ } else if ((isForceMinorCompactionAllow || enableMinorCompaction) &&
(!suspendMinor)
&& (force || curTime - lastMinorCompactionTime >
minorCompactionInterval)) {
// enter minor compaction
LOG.info("Enter minor compaction, suspendMinor {}", suspendMinor);
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index da74805..6b6995f 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -90,6 +90,7 @@ public class ServerConfiguration extends
AbstractConfiguration<ServerConfigurati
protected static final String ENTRY_LOG_FILE_PREALLOCATION_ENABLED =
"entryLogFilePreallocationEnabled";
+ protected static final String FORCE_ALLOW_COMPACTION =
"forceAllowCompaction";
protected static final String MINOR_COMPACTION_INTERVAL =
"minorCompactionInterval";
protected static final String MINOR_COMPACTION_THRESHOLD =
"minorCompactionThreshold";
protected static final String MAJOR_COMPACTION_INTERVAL =
"majorCompactionInterval";
@@ -1451,6 +1452,27 @@ public class ServerConfiguration extends
AbstractConfiguration<ServerConfigurati
}
/**
+ * Allow manually force compact the entry log or not.
+ *
+ * @param enable
+ * whether allow manually force compact the entry log or not.
+ * @return service configuration.
+ */
+ public ServerConfiguration setForceAllowCompaction(boolean enable) {
+ setProperty(FORCE_ALLOW_COMPACTION, enable);
+ return this;
+ }
+
+ /**
+ * The force compaction is allowed or not when disabling the entry log
compaction.
+ *
+ * @return the force compaction is allowed or not when disabling the entry
log compaction.
+ */
+ public boolean isForceAllowCompaction() {
+ return getBoolean(FORCE_ALLOW_COMPACTION, false);
+ }
+
+ /**
* Get threshold of minor compaction.
*
* <p>For those entry log files whose remaining size percentage reaches
below
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
index 8fd9c3a..0b6f2e4 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
@@ -229,11 +229,45 @@ public abstract class CompactionTest extends
BookKeeperClusterTestCase {
}
@Test
+ public void
testForceGarbageCollectionWhenDisableCompactionConfigurationSettings() throws
Exception {
+ // prepare data
+ LedgerHandle[] lhs = prepareData(3, false);
+
+ baseConf.setForceAllowCompaction(true);
+ baseConf.setMajorCompactionThreshold(0.5f);
+ baseConf.setMinorCompactionThreshold(0.2f);
+ baseConf.setMajorCompactionInterval(0);
+ baseConf.setMinorCompactionInterval(0);
+ restartBookies(baseConf);
+
+ assertFalse(getGCThread().enableMajorCompaction);
+ assertFalse(getGCThread().enableMinorCompaction);
+ assertTrue(getGCThread().isForceMajorCompactionAllow);
+ assertTrue(getGCThread().isForceMinorCompactionAllow);
+
+ assertEquals(0.5f, getGCThread().majorCompactionThreshold, 0f);
+ assertEquals(0.2f, getGCThread().minorCompactionThreshold, 0f);
+ }
+
+ @Test
public void testForceGarbageCollection() throws Exception {
+ testForceGarbageCollection(true);
+ testForceGarbageCollection(false);
+ }
+
+ public void testForceGarbageCollection(boolean
isForceCompactionAllowWhenDisableCompaction) throws Exception {
ServerConfiguration conf = newServerConfiguration();
conf.setGcWaitTime(60000);
- conf.setMinorCompactionInterval(120000);
- conf.setMajorCompactionInterval(240000);
+ if (isForceCompactionAllowWhenDisableCompaction) {
+ conf.setMinorCompactionInterval(0);
+ conf.setMajorCompactionInterval(0);
+ conf.setForceAllowCompaction(true);
+ conf.setMajorCompactionThreshold(0.5f);
+ conf.setMinorCompactionThreshold(0.2f);
+ } else {
+ conf.setMinorCompactionInterval(120000);
+ conf.setMajorCompactionInterval(240000);
+ }
LedgerDirsManager dirManager = new LedgerDirsManager(conf,
conf.getLedgerDirs(),
new DiskChecker(conf.getDiskUsageThreshold(),
conf.getDiskUsageWarnThreshold()));
CheckpointSource cp = new CheckpointSource() {
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index 1bcff80..c6e6937 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -476,7 +476,13 @@ ledgerDirectories=/tmp/bk-data
## Entry log compaction settings
#############################################################################
-# Set the rate at which compaction will readd entries. The unit is adds per
second.
+# Allow force compaction when disabling the entry log compaction or not.
+# It will enable you to manually force compact the entry log even if
+# the entry log compaction is disabled. The 'minorCompactionThreshold' or
+# 'majorCompactionThreshold' still needs to be specified.
+# forceAllowCompaction=false
+
+# Set the rate at which compaction will read entries. The unit is adds per
second.
# compactionRate=1000
# Threshold of minor compaction
