This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push: new 34746858b4 Added flag to control whether to transition to read-only mode when any disks full (#3212) 34746858b4 is described below commit 34746858b47e3962914f646803e930afe77859a3 Author: Hang Chen <chenh...@apache.org> AuthorDate: Sat Jul 23 00:59:42 2022 +0800 Added flag to control whether to transition to read-only mode when any disks full (#3212) * Added flag to control whether to transition to read-only mode when any disk is full * add unit test * address comments * add docs and config in conf/bk_server.conf --- .../org/apache/bookkeeper/bookie/BookieImpl.java | 21 +++++ .../bookkeeper/bookie/LedgerDirsManager.java | 13 ++++ .../bookkeeper/bookie/LedgerDirsMonitor.java | 26 ++++++- .../bookkeeper/conf/ServerConfiguration.java | 22 ++++++ .../bookkeeper/bookie/LedgerDirsManagerTest.java | 90 ++++++++++++++++++++++ conf/bk_server.conf | 9 ++- site3/website/docs/reference/config.md | 6 +- 7 files changed, 181 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index 9bc084ee92..4dc717e9bb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -741,6 +741,9 @@ public class BookieImpl extends BookieCriticalThread implements Bookie { @Override public void diskWritable(File disk) { + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + return; + } // Transition to writable mode when a disk becomes writable again. stateManager.setHighPriorityWritesAvailability(true); stateManager.transitionToWritableMode(); @@ -748,6 +751,24 @@ public class BookieImpl extends BookieCriticalThread implements Bookie { @Override public void diskJustWritable(File disk) { + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + return; + } + // Transition to writable mode when a disk becomes writable again. + stateManager.setHighPriorityWritesAvailability(true); + stateManager.transitionToWritableMode(); + } + + @Override + public void anyDiskFull(boolean highPriorityWritesAllowed) { + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + stateManager.setHighPriorityWritesAvailability(highPriorityWritesAllowed); + stateManager.transitionToReadOnlyMode(); + } + } + + @Override + public void allDisksWritable() { // Transition to writable mode when a disk becomes writable again. stateManager.setHighPriorityWritesAvailability(true); stateManager.transitionToWritableMode(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java index a9d438e4b7..b9934a2ba7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java @@ -434,6 +434,19 @@ public class LedgerDirsManager { */ default void allDisksFull(boolean highPriorityWritesAllowed) {} + /** + * This will be notified whenever all disks are detected as not full. + * + */ + default void allDisksWritable() {} + + /** + * This will be notified whenever any disks are detected as full. + * + * @param highPriorityWritesAllowed the parameter indicates we are still have disk spaces for high priority + * * writes even disks are detected as "full" + */ + default void anyDiskFull(boolean highPriorityWritesAllowed) {} /** * This will notify the fatal errors. */ diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java index 2b7c90152d..f7c2dc31ba 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java @@ -52,7 +52,7 @@ class LedgerDirsMonitor { private final ServerConfiguration conf; private final DiskChecker diskChecker; private final List<LedgerDirsManager> dirsManagers; - private long minUsableSizeForHighPriorityWrites; + private final long minUsableSizeForHighPriorityWrites; private ScheduledExecutorService executor; private ScheduledFuture<?> checkTask; @@ -68,6 +68,10 @@ class LedgerDirsMonitor { private void check(final LedgerDirsManager ldm) { final ConcurrentMap<File, Float> diskUsages = ldm.getDiskUsages(); + boolean someDiskFulled = false; + boolean highPriorityWritesAllowed = true; + boolean someDiskRecovered = false; + try { List<File> writableDirs = ldm.getWritableLedgerDirs(); // Check all writable dirs disk space usage. @@ -99,6 +103,7 @@ class LedgerDirsMonitor { }); // Notify disk full to all listeners ldm.addToFilledDirs(dir); + someDiskFulled = true; } } // Let's get NoWritableLedgerDirException without waiting for the next iteration @@ -108,7 +113,6 @@ class LedgerDirsMonitor { ldm.getWritableLedgerDirs(); } catch (NoWritableLedgerDirException e) { LOG.warn("LedgerDirsMonitor check process: All ledger directories are non writable"); - boolean highPriorityWritesAllowed = true; try { // disk check can be frequent, so disable 'loggingNoWritable' to avoid log flooding. ldm.getDirsAboveUsableThresholdSize(minUsableSizeForHighPriorityWrites, false); @@ -146,6 +150,7 @@ class LedgerDirsMonitor { if (makeWritable) { ldm.addToWritableDirs(dir, true); } + someDiskRecovered = true; } catch (DiskErrorException e) { // Notify disk failure to all the listeners for (LedgerDirsListener listener : ldm.getListeners()) { @@ -157,6 +162,7 @@ class LedgerDirsMonitor { if (makeWritable) { ldm.addToWritableDirs(dir, false); } + someDiskRecovered = true; } catch (DiskOutOfSpaceException e) { // the full-filled dir is still full-filled diskUsages.put(dir, e.getUsage()); @@ -168,6 +174,22 @@ class LedgerDirsMonitor { listener.fatalError(); } } + + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + if (someDiskFulled && !ldm.getFullFilledLedgerDirs().isEmpty()) { + // notify any disk full. + for (LedgerDirsListener listener : ldm.getListeners()) { + listener.anyDiskFull(highPriorityWritesAllowed); + } + } + + if (someDiskRecovered && ldm.getFullFilledLedgerDirs().isEmpty()) { + // notify all disk recovered. + for (LedgerDirsListener listener : ldm.getListeners()) { + listener.allDisksWritable(); + } + } + } } private void check() { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index fa9d58b1da..5518bf0362 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -187,6 +187,7 @@ public class ServerConfiguration extends AbstractConfiguration<ServerConfigurati protected static final String LOCK_RELEASE_OF_FAILED_LEDGER_GRACE_PERIOD = "lockReleaseOfFailedLedgerGracePeriod"; //ReadOnly mode support on all disk full protected static final String READ_ONLY_MODE_ENABLED = "readOnlyModeEnabled"; + protected static final String READ_ONLY_MODE_ON_ANY_DISK_FULL_ENABLED = "readOnlyModeOnAnyDiskFullEnabled"; //Whether the bookie is force started in ReadOnly mode protected static final String FORCE_READ_ONLY_BOOKIE = "forceReadOnlyBookie"; //Whether to persist the bookie status @@ -2395,6 +2396,27 @@ public class ServerConfiguration extends AbstractConfiguration<ServerConfigurati return getBoolean(READ_ONLY_MODE_ENABLED, true); } + /** + * Set whether the bookie is able to go into read-only mode when any disk is full. + * If this set to false, it will behave to READ_ONLY_MODE_ENABLED flag. + * + * @param enabled whether to enable read-only mode when any disk is full. + * @return + */ + public ServerConfiguration setReadOnlyModeOnAnyDiskFullEnabled(boolean enabled) { + setProperty(READ_ONLY_MODE_ON_ANY_DISK_FULL_ENABLED, enabled); + return this; + } + + /** + * Get whether read-only mode is enable when any disk is full. The default is false. + * + * @return boolean + */ + public boolean isReadOnlyModeOnAnyDiskFullEnabled() { + return getBoolean(READ_ONLY_MODE_ON_ANY_DISK_FULL_ENABLED, false); + } + /** * Set the warning threshold for disk usage. * diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java index 31b6da37a3..91d3ba2b34 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java @@ -240,6 +240,76 @@ public class LedgerDirsManagerTest { assertTrue(mockLedgerDirsListener.highPriorityWritesAllowed); } + @Test + public void testIsReadOnlyModeOnAnyDiskFullEnabled() throws Exception { + testAnyLedgerFullTransitToReadOnly(true); + testAnyLedgerFullTransitToReadOnly(false); + } + + public void testAnyLedgerFullTransitToReadOnly(boolean isReadOnlyModeOnAnyDiskFullEnabled) throws Exception { + ledgerMonitor.shutdown(); + + final float nospace = 0.90f; + final float lwm = 0.80f; + HashMap<File, Float> usageMap; + + File tmpDir1 = createTempDir("bkTest", ".dir"); + File curDir1 = BookieImpl.getCurrentDirectory(tmpDir1); + BookieImpl.checkDirectoryStructure(curDir1); + + File tmpDir2 = createTempDir("bkTest", ".dir"); + File curDir2 = BookieImpl.getCurrentDirectory(tmpDir2); + BookieImpl.checkDirectoryStructure(curDir2); + + conf.setDiskUsageThreshold(nospace); + conf.setDiskLowWaterMarkUsageThreshold(lwm); + conf.setDiskUsageWarnThreshold(nospace); + conf.setReadOnlyModeOnAnyDiskFullEnabled(isReadOnlyModeOnAnyDiskFullEnabled); + conf.setLedgerDirNames(new String[] { tmpDir1.toString(), tmpDir2.toString() }); + + mockDiskChecker = new MockDiskChecker(nospace, warnThreshold); + dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), + new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()), statsLogger); + ledgerMonitor = new LedgerDirsMonitor(conf, mockDiskChecker, Collections.singletonList(dirsManager)); + usageMap = new HashMap<>(); + usageMap.put(curDir1, 0.1f); + usageMap.put(curDir2, 0.1f); + mockDiskChecker.setUsageMap(usageMap); + ledgerMonitor.init(); + final MockLedgerDirsListener mockLedgerDirsListener = new MockLedgerDirsListener(); + dirsManager.addLedgerDirsListener(mockLedgerDirsListener); + ledgerMonitor.start(); + + Thread.sleep((diskCheckInterval * 2) + 100); + assertFalse(mockLedgerDirsListener.readOnly); + + if (isReadOnlyModeOnAnyDiskFullEnabled) { + setUsageAndThenVerify(curDir1, 0.1f, curDir2, nospace + 0.05f, mockDiskChecker, + mockLedgerDirsListener, true); + setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, 0.1f, mockDiskChecker, + mockLedgerDirsListener, true); + setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, nospace + 0.05f, mockDiskChecker, + mockLedgerDirsListener, true); + setUsageAndThenVerify(curDir1, nospace - 0.30f, curDir2, nospace + 0.05f, mockDiskChecker, + mockLedgerDirsListener, true); + setUsageAndThenVerify(curDir1, nospace - 0.20f, curDir2, nospace - 0.20f, mockDiskChecker, + mockLedgerDirsListener, false); + } else { + setUsageAndThenVerify(curDir1, 0.1f, curDir2, 0.1f, mockDiskChecker, + mockLedgerDirsListener, false); + setUsageAndThenVerify(curDir1, 0.1f, curDir2, nospace + 0.05f, mockDiskChecker, + mockLedgerDirsListener, false); + setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, 0.1f, mockDiskChecker, + mockLedgerDirsListener, false); + setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, nospace + 0.05f, mockDiskChecker, + mockLedgerDirsListener, true); + setUsageAndThenVerify(curDir1, nospace - 0.30f, curDir2, nospace + 0.05f, mockDiskChecker, + mockLedgerDirsListener, false); + setUsageAndThenVerify(curDir1, nospace - 0.20f, curDir2, nospace - 0.20f, mockDiskChecker, + mockLedgerDirsListener, false); + } + } + @Test public void testLedgerDirsMonitorHandlingLowWaterMark() throws Exception { ledgerMonitor.shutdown(); @@ -517,12 +587,18 @@ public class LedgerDirsManagerTest { @Override public void diskWritable(File disk) { + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + return; + } readOnly = false; highPriorityWritesAllowed = true; } @Override public void diskJustWritable(File disk) { + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + return; + } readOnly = false; highPriorityWritesAllowed = true; } @@ -533,6 +609,20 @@ public class LedgerDirsManagerTest { this.highPriorityWritesAllowed = highPriorityWritesAllowed; } + @Override + public void anyDiskFull(boolean highPriorityWritesAllowed) { + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { + this.readOnly = true; + this.highPriorityWritesAllowed = highPriorityWritesAllowed; + } + } + + @Override + public void allDisksWritable() { + this.readOnly = false; + this.highPriorityWritesAllowed = true; + } + public void reset() { readOnly = false; highPriorityWritesAllowed = true; diff --git a/conf/bk_server.conf b/conf/bk_server.conf index ac6fc8aa47..9890674ca4 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -170,7 +170,7 @@ extraServerComponents= # If all ledger directories configured are full, then support only read requests for clients. # If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted # to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. -# By default this will be disabled. +# By default this will be enabled. # readOnlyModeEnabled=true # Whether the bookie is force started in read only mode or not @@ -180,6 +180,13 @@ extraServerComponents= # @Since 4.6 # persistBookieStatusEnabled=false +# If any ledger directories configured are full, then support only read requests for clients. +# If "readOnlyModeOnAnyDiskFullEnabled=true" then on any ledger disks full, bookie will be converted +# to read-only mode and serve only read requests. When all disks recovered, +# the bookie will be converted to read-write mode.Otherwise it will obey the `readOnlyModeEnabled` behavior. +# By default this will be disabled. +# readOnlyModeOnAnyDiskFullEnabled=false + ############################################################################# ## Netty server settings ############################################################################# diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md index 891b447421..57d9abd89b 100644 --- a/site3/website/docs/reference/config.md +++ b/site3/website/docs/reference/config.md @@ -51,11 +51,11 @@ The table below lists parameters that you can set to configure bookies. All conf ## Read-only mode support | Parameter | Description | Default -| --------- | ----------- | ------- | -| readOnlyModeEnabled | If all ledger directories configured are full, then support only read requests for clients. If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. By default this will be disabled. | true | +| -- | ----------- | ------- | +| readOnlyModeEnabled | If all ledger directories configured are full, then support only read requests for clients. If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. By default, this will be enabled. | true | | forceReadOnlyBookie | Whether the bookie is force started in read only mode or not. | false | | persistBookieStatusEnabled | Persist the bookie status locally on the disks. So the bookies can keep their status upon restarts. | false | - +| readOnlyModeOnAnyDiskFullEnabled | If any ledger directories configured are full, then support only read requests for clients. If "readOnlyModeOnAnyDiskFullEnabled=true" then on any ledger disks full, bookie will be converted to read-only mode and serve only read requests. When all disks recovered, the bookie will be converted to read-write mode.Otherwise it will obey the `readOnlyModeEnabled` behavior. By default, this will be disabled. | false | ## Netty server settings