This is an automated email from the ASF dual-hosted git repository.
shoothzj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1fcf74c1dc stage 2.3 ~ 2.4: (#3397)
1fcf74c1dc is described below
commit 1fcf74c1dc18cfc96960ebf41170177f5a98e2d6
Author: StevenLuMT <[email protected]>
AuthorDate: Tue Jul 12 06:58:38 2022 +0800
stage 2.3 ~ 2.4: (#3397)
Descriptions of the changes in this PR:
### Motivation
planning for index dir: [mail
talking](https://lists.apache.org/thread/r657jf55khl59bbqltj2s95107lbkr0w)
stage 3~4 :
1) disk monitor(DiskChecker/LedgerDirsMonitor) have checked indexDir,it's
good;
2) add indexDirs in statsLogger, and add indexDirs's Listener.
### Changes
1. add indexDirs in statsLogger, and add indexDirs's Listener.
2. update index's testcase for SingleDirectoryDbLedgerStorage
---
.../ldb/SingleDirectoryDbLedgerStorage.java | 31 ++++++++++++++--------
.../storage/ldb/DbLedgerStorageIndexDirTest.java | 9 ++++++-
2 files changed, 28 insertions(+), 12 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
index 9ac56b9649..a1131df815 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
@@ -80,8 +80,8 @@ import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.ThreadRegistry;
import org.apache.bookkeeper.util.MathUtils;
import org.apache.bookkeeper.util.collections.ConcurrentLongHashMap;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang.mutable.MutableLong;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -152,17 +152,21 @@ public class SingleDirectoryDbLedgerStorage implements
CompactableLedgerStorage
checkArgument(ledgerDirsManager.getAllLedgerDirs().size() == 1,
"Db implementation only allows for one storage dir");
- String indexBaseDir =
indexDirsManager.getAllLedgerDirs().get(0).toString();
- if (StringUtils.isBlank(indexBaseDir)) {
- indexBaseDir =
ledgerDirsManager.getAllLedgerDirs().get(0).toString();
+ String ledgerBaseDir =
ledgerDirsManager.getAllLedgerDirs().get(0).getPath();
+ // indexBaseDir default use ledgerBaseDir
+ String indexBaseDir = ledgerBaseDir;
+ if (CollectionUtils.isEmpty(indexDirsManager.getAllLedgerDirs())) {
log.info("indexDir is not specified, use default, creating single
directory db ledger storage on {}",
indexBaseDir);
} else {
+ // if indexDir is specified, set new value
+ indexBaseDir =
indexDirsManager.getAllLedgerDirs().get(0).getPath();
log.info("indexDir is specified, creating single directory db
ledger storage on {}", indexBaseDir);
}
- StatsLogger ledgerDirStatsLogger = statsLogger.scopeLabel("ledgerDir",
- ledgerDirsManager.getAllLedgerDirs().get(0).getPath());
+ StatsLogger ledgerIndexDirStatsLogger = statsLogger
+ .scopeLabel("ledgerDir", ledgerBaseDir)
+ .scopeLabel("indexDir", indexBaseDir);
this.writeCacheMaxSize = writeCacheSize;
this.writeCache = new WriteCache(allocator, writeCacheMaxSize / 2);
@@ -180,9 +184,10 @@ public class SingleDirectoryDbLedgerStorage implements
CompactableLedgerStorage
readCache = new ReadCache(allocator, readCacheMaxSize);
- ledgerIndex = new LedgerMetadataIndex(conf,
KeyValueStorageRocksDB.factory, indexBaseDir, ledgerDirStatsLogger);
+ ledgerIndex = new LedgerMetadataIndex(conf,
+ KeyValueStorageRocksDB.factory, indexBaseDir,
ledgerIndexDirStatsLogger);
entryLocationIndex = new EntryLocationIndex(conf,
- KeyValueStorageRocksDB.factory, indexBaseDir,
ledgerDirStatsLogger);
+ KeyValueStorageRocksDB.factory, indexBaseDir,
ledgerIndexDirStatsLogger);
transientLedgerInfoCache =
ConcurrentLongHashMap.<TransientLedgerInfo>newBuilder()
.expectedItems(16 * 1024)
@@ -193,17 +198,18 @@ public class SingleDirectoryDbLedgerStorage implements
CompactableLedgerStorage
TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES,
TimeUnit.MINUTES);
this.entryLogger = entryLogger;
- gcThread = new GarbageCollectorThread(conf, ledgerManager,
ledgerDirsManager, this, entryLogger, statsLogger);
+ gcThread = new GarbageCollectorThread(conf,
+ ledgerManager, ledgerDirsManager, this, entryLogger,
ledgerIndexDirStatsLogger);
dbLedgerStorageStats = new DbLedgerStorageStats(
- ledgerDirStatsLogger,
+ ledgerIndexDirStatsLogger,
() -> writeCache.size() + writeCacheBeingFlushed.size(),
() -> writeCache.count() + writeCacheBeingFlushed.count(),
() -> readCache.size(),
() -> readCache.count()
);
- flushExecutorTime =
ledgerDirStatsLogger.getThreadScopedCounter("db-storage-thread-time");
+ flushExecutorTime =
ledgerIndexDirStatsLogger.getThreadScopedCounter("db-storage-thread-time");
executor.submit(() -> {
ThreadRegistry.register(dbStoragerExecutorName, 0);
@@ -213,6 +219,9 @@ public class SingleDirectoryDbLedgerStorage implements
CompactableLedgerStorage
});
ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());
+ if (!ledgerBaseDir.equals(indexBaseDir)) {
+ indexDirsManager.addLedgerDirsListener(getLedgerDirsListener());
+ }
}
@Override
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
index 1f8e64c4cb..961f340c81 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
@@ -69,6 +69,7 @@ public class DbLedgerStorageIndexDirTest {
int gcWaitTime = 1000;
ServerConfiguration conf =
TestBKConfiguration.newServerConfiguration();
conf.setGcWaitTime(gcWaitTime);
+ /** the testcase cover specify indexDir for the class {@link
SingleDirectoryDbLedgerStorage} */
conf.setLedgerStorageClass(DbLedgerStorage.class.getName());
conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 1);
conf.setProperty(DbLedgerStorage.MAX_THROTTLE_TIME_MILLIS, 1000);
@@ -124,8 +125,14 @@ public class DbLedgerStorageIndexDirTest {
}
@Test
- public void checkIndexDirectoryStructure() {
+ public void checkIndexNotExistsInLedgerDirStructure() {
+ // old logic bugfix
assertEquals(false, hasIndexStructure(tmpLedgerDir));
+ }
+
+ @Test
+ public void checkIndexDirectoryStructure() {
+ // index new logic
assertEquals(true, hasIndexStructure(tmpIndexDir));
}
