This is an automated email from the ASF dual-hosted git repository.
chenhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new dd80e967e7 [fix][server] When dbStorage_directIOEntryLogger=true,
EntryLogIdsImpl only takes effect for a single ledger directory (#3680)
dd80e967e7 is described below
commit dd80e967e7d3dc6873231a7bc04feaba83be54c3
Author: LinChen <[email protected]>
AuthorDate: Thu Dec 22 09:48:52 2022 +0800
[fix][server] When dbStorage_directIOEntryLogger=true, EntryLogIdsImpl only
takes effect for a single ledger directory (#3680)
### Motivation
When building the EntryLogIdsImpl object, the ledgerDirsManager variable is
used. When the EntryLogIdsImpl object is used to generate an id, all ledger
directories will be scanned in the findLargestGap method. for example:
We configured three ledger directories:
/data1/bk-data1
/data1/bk-data2
/data1/bk-data3
The findLargestGap method scans these three data directories to generate
ids, but actually only needs to scan one data directory corresponding to the
EntryLogIdsImpl object:
https://github.com/apache/bookkeeper/blob/01232c94c91759a175345e8c055951e5cc6091dd/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java#L225-L236
### Changes
When building the EntryLogIdsImpl object, use the variable ldm instead of
the variable ledgerDirsManager:
```
lDirs[0] = ledgerDir.getParentFile();
LedgerDirsManager ldm = new LedgerDirsManager(conf, lDirs,
ledgerDirsManager.getDiskChecker(),
NullStatsLogger.INSTANCE);
```
```
entrylogger = new DirectEntryLogger(ledgerDir, new EntryLogIdsImpl(ldm,
slog),
new NativeIOImpl(),
allocator, entryLoggerWriteExecutor,
entryLoggerFlushExecutor,
conf.getEntryLogSizeLimit(),
conf.getNettyMaxFrameSizeBytes() - 500,
perDirectoryTotalWriteBufferSize,
perDirectoryTotalReadBufferSize,
readBufferSize,
numReadThreads,
maxFdCacheTimeSeconds,
slog, statsLogger);
```
---
.../bookie/storage/ldb/DbLedgerStorage.java | 2 +-
.../bookkeeper/bookie/storage/TestEntryLogIds.java | 47 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
index df071acb67..31697067a1 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
@@ -217,7 +217,7 @@ public class DbLedgerStorage implements LedgerStorage {
numReadThreads = conf.getServerNumIOThreads();
}
- entrylogger = new DirectEntryLogger(ledgerDir, new
EntryLogIdsImpl(ledgerDirsManager, slog),
+ entrylogger = new DirectEntryLogger(ledgerDir, new
EntryLogIdsImpl(ldm, slog),
new NativeIOImpl(),
allocator, entryLoggerWriteExecutor,
entryLoggerFlushExecutor,
conf.getEntryLogSizeLimit(),
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/TestEntryLogIds.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/TestEntryLogIds.java
index 54701cbcf1..cee1365f0a 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/TestEntryLogIds.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/TestEntryLogIds.java
@@ -32,6 +32,7 @@ import static org.junit.Assert.assertEquals;
import com.google.common.collect.Lists;
import io.netty.buffer.ByteBuf;
import java.io.File;
+import org.apache.bookkeeper.bookie.LedgerDirsManager;
import
org.apache.bookkeeper.bookie.storage.directentrylogger.DirectCompactionEntryLog;
import
org.apache.bookkeeper.bookie.storage.directentrylogger.DirectEntryLogger;
import org.apache.bookkeeper.slogger.Slogger;
@@ -127,6 +128,52 @@ public class TestEntryLogIds {
highestSoFar = logId4;
}
+ @Test
+ public void testIdGenerator() throws Exception {
+ File base = tmpDirs.createNew("entryLogIds", "ledgers");
+ File ledgerDir1 = new File(base, "l1");
+ File ledgerDir2 = new File(base, "l2");
+ File ledgerDir3 = new File(base, "l3");
+ File ledgerDir4 = new File(base, "l4");
+ ledgerDir1.mkdir();
+ ledgerDir2.mkdir();
+ ledgerDir3.mkdir();
+ ledgerDir4.mkdir();
+
+ //case 1: use root ledgerDirsManager
+ LedgerDirsManager ledgerDirsManager = newDirsManager(ledgerDir1,
ledgerDir2);
+ EntryLogIds ids1 = new EntryLogIdsImpl(ledgerDirsManager, slog);
+ for (int i = 0; i < 10; i++) {
+ int logId = ids1.nextId();
+ File log1 = new File(ledgerDir1 + "/current", logId + ".log");
+ log1.createNewFile();
+ assertEquals(logId, i);
+ }
+
+ EntryLogIds ids2 = new EntryLogIdsImpl(ledgerDirsManager, slog);
+ for (int i = 0; i < 10; i++) {
+ int logId = ids2.nextId();
+ assertEquals(logId, 10 + i);
+ }
+
+ // case 2: new LedgerDirsManager for per directory
+ LedgerDirsManager ledgerDirsManager3 = newDirsManager(ledgerDir3);
+ LedgerDirsManager ledgerDirsManager4 = newDirsManager(ledgerDir4);
+ EntryLogIds ids3 = new EntryLogIdsImpl(ledgerDirsManager3, slog);
+ for (int i = 0; i < 10; i++) {
+ int logId = ids3.nextId();
+ File log1 = new File(ledgerDir3 + "/current", logId + ".log");
+ log1.createNewFile();
+ assertEquals(logId, i);
+ }
+
+ EntryLogIds ids4 = new EntryLogIdsImpl(ledgerDirsManager4, slog);
+ for (int i = 0; i < 10; i++) {
+ int logId = ids4.nextId();
+ assertEquals(logId, i);
+ }
+ }
+
@Test
public void testMultiDirectory() throws Exception {
File base = tmpDirs.createNew("entryLogIds", "ledgers");
