Repository: bookkeeper Updated Branches: refs/heads/master c8255f8c5 -> bfa74f736
Bookie should return to read-write mode once the disk usage drops before the threshold â¦o read only mode Author: Siddharth Boobna <[email protected]> Reviewers: Sijie Guo <[email protected]> Closes #18 from sboobna/BOOKKEEPER-899 Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/bfa74f73 Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/bfa74f73 Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/bfa74f73 Branch: refs/heads/master Commit: bfa74f736ba270cc1a0bd0f7a1a6c643a00e385d Parents: c8255f8 Author: Siddharth Boobna <[email protected]> Authored: Mon Mar 7 22:14:10 2016 -0800 Committer: Sijie Guo <[email protected]> Committed: Mon Mar 7 22:14:10 2016 -0800 ---------------------------------------------------------------------- .../bookkeeper/bookie/LedgerDirsManager.java | 64 ++++++----- .../org/apache/bookkeeper/util/DiskChecker.java | 8 ++ .../bookie/TestLedgerDirsManager.java | 115 ++++++++++++++++++- 3 files changed, 151 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bfa74f73/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java index 935befb..7c95b85 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java @@ -66,13 +66,18 @@ public class LedgerDirsManager { } LedgerDirsManager(ServerConfiguration conf, File[] dirs, StatsLogger statsLogger) { + this(conf, dirs, statsLogger, new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold())); + } + + @VisibleForTesting + LedgerDirsManager(ServerConfiguration conf, File[] dirs, StatsLogger statsLogger, DiskChecker diskChecker) { this.ledgerDirectories = Arrays.asList(Bookie .getCurrentDirectories(dirs)); this.writableLedgerDirectories = new ArrayList<File>(ledgerDirectories); this.filledDirs = new ArrayList<File>(); - listeners = new ArrayList<LedgerDirsListener>(); - diskChecker = new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()); - monitor = new LedgerDirsMonitor(conf.getDiskCheckInterval()); + this.listeners = new ArrayList<LedgerDirsListener>(); + this.diskChecker = diskChecker; + this.monitor = new LedgerDirsMonitor(conf.getDiskCheckInterval()); for (File dir : dirs) { diskUsages.put(dir, 0f); String statName = "dir_" + dir.getPath().replace('/', '_') + "_usage"; @@ -274,38 +279,37 @@ public class LedgerDirsManager { @Override public void run() { while (true) { - List<File> writableDirs; try { - writableDirs = getWritableLedgerDirs(); + List<File> writableDirs = getWritableLedgerDirs(); + // Check all writable dirs disk space usage. + for (File dir : writableDirs) { + try { + diskUsages.put(dir, diskChecker.checkDir(dir)); + } catch (DiskErrorException e) { + LOG.error("Ledger directory {} failed on disk checking : ", dir, e); + // Notify disk failure to all listeners + for (LedgerDirsListener listener : listeners) { + listener.diskFailed(dir); + } + } catch (DiskWarnThresholdException e) { + LOG.warn("Ledger directory {} is almost full.", dir); + diskUsages.put(dir, e.getUsage()); + for (LedgerDirsListener listener : listeners) { + listener.diskAlmostFull(dir); + } + } catch (DiskOutOfSpaceException e) { + LOG.error("Ledger directory {} is out-of-space.", dir); + diskUsages.put(dir, e.getUsage()); + // Notify disk full to all listeners + addToFilledDirs(dir); + } + } } catch (NoWritableLedgerDirException e) { for (LedgerDirsListener listener : listeners) { listener.allDisksFull(); } - break; - } - // Check all writable dirs disk space usage. - for (File dir : writableDirs) { - try { - diskUsages.put(dir, diskChecker.checkDir(dir)); - } catch (DiskErrorException e) { - LOG.error("Ledger directory {} failed on disk checking : ", dir, e); - // Notify disk failure to all listeners - for (LedgerDirsListener listener : listeners) { - listener.diskFailed(dir); - } - } catch (DiskWarnThresholdException e) { - LOG.warn("Ledger directory {} is almost full.", dir); - diskUsages.put(dir, e.getUsage()); - for (LedgerDirsListener listener : listeners) { - listener.diskAlmostFull(dir); - } - } catch (DiskOutOfSpaceException e) { - LOG.error("Ledger directory {} is out-of-space.", dir); - diskUsages.put(dir, e.getUsage()); - // Notify disk full to all listeners - addToFilledDirs(dir); - } } + List<File> fullfilledDirs = new ArrayList<File>(getFullFilledLedgerDirs()); // Check all full-filled disk space usage for (File dir : fullfilledDirs) { @@ -313,7 +317,7 @@ public class LedgerDirsManager { diskUsages.put(dir, diskChecker.checkDir(dir)); addToWritableDirs(dir, true); } catch (DiskErrorException e) { - //Notify disk failure to all the listeners + // Notify disk failure to all the listeners for (LedgerDirsListener listener : listeners) { listener.diskFailed(dir); } http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bfa74f73/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java index 75902c9..b1251a6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java @@ -205,4 +205,12 @@ public class DiskChecker { + " are not valid. Should be > 0 and < 1 and diskSpaceThreshold >= diskSpaceWarnThreshold"); } } + + public float getDiskUsageThreshold() { + return diskUsageThreshold; + } + + public float getDiskUsageWarnThreshold() { + return diskUsageWarnThreshold; + } } http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bfa74f73/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java index 416062d..4d8d547 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java @@ -20,30 +20,40 @@ */ package org.apache.bookkeeper.bookie; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.commons.io.FileUtils; +import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener; import org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.util.DiskChecker; import org.apache.bookkeeper.util.IOUtils; -import org.junit.Before; +import org.apache.commons.io.FileUtils; import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.*; - public class TestLedgerDirsManager { private final static Logger LOG = LoggerFactory.getLogger(TestLedgerDirsManager.class); ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); File curDir; LedgerDirsManager dirsManager; + MockDiskChecker mockDiskChecker; + int diskCheckInterval = 1000; + float threshold = 0.5f; + float warnThreshold = 0.5f; final List<File> tempDirs = new ArrayList<File>(); @@ -60,13 +70,17 @@ public class TestLedgerDirsManager { Bookie.checkDirectoryStructure(curDir); ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); - conf.setLedgerDirNames(new String[] {tmpDir.toString()}); + conf.setLedgerDirNames(new String[] { tmpDir.toString() }); + conf.setDiskCheckInterval(diskCheckInterval); - dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs()); + mockDiskChecker = new MockDiskChecker(threshold, warnThreshold); + dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), NullStatsLogger.INSTANCE, mockDiskChecker); + dirsManager.init(); } @After public void tearDown() throws Exception { + dirsManager.shutdown(); for (File dir : tempDirs) { FileUtils.deleteDirectory(dir); } @@ -97,4 +111,93 @@ public class TestLedgerDirsManager { } } + @Test(timeout=60000) + public void testLedgerDirsMonitorDuringTransition() throws Exception { + + MockLedgerDirsListener mockLedgerDirsListener = new MockLedgerDirsListener(); + dirsManager.addLedgerDirsListener(mockLedgerDirsListener); + dirsManager.start(); + + assertFalse(mockLedgerDirsListener.readOnly); + mockDiskChecker.setUsage(threshold + 0.05f); + + Thread.sleep((diskCheckInterval * 2) + 100); + + assertTrue(mockLedgerDirsListener.readOnly); + mockDiskChecker.setUsage(threshold - 0.05f); + + Thread.sleep(diskCheckInterval + 100); + + assertFalse(mockLedgerDirsListener.readOnly); + } + + private class MockDiskChecker extends DiskChecker { + + private float used; + + public MockDiskChecker(float threshold, float warnThreshold) { + super(threshold, warnThreshold); + used = 0f; + } + + @Override + public float checkDir(File dir) throws DiskErrorException, DiskOutOfSpaceException, DiskWarnThresholdException { + if (used > getDiskUsageThreshold()) { + throw new DiskOutOfSpaceException("", used); + } + if (used > getDiskUsageWarnThreshold()) { + throw new DiskWarnThresholdException("", used); + } + return used; + } + + public void setUsage(float usage) { + this.used = usage; + } + } + + private class MockLedgerDirsListener implements LedgerDirsListener { + + public boolean readOnly; + + public MockLedgerDirsListener() { + reset(); + } + + @Override + public void diskFailed(File disk) { + } + + @Override + public void diskAlmostFull(File disk) { + } + + @Override + public void diskFull(File disk) { + } + + @Override + public void diskWritable(File disk) { + readOnly = false; + } + + @Override + public void diskJustWritable(File disk) { + readOnly = false; + } + + @Override + public void allDisksFull() { + readOnly = true; + } + + @Override + public void fatalError() { + } + + public void reset() { + readOnly = false; + } + + } }
