bookkeeper git commit: BOOKKEEPER-850: Use nanoseconds to calculate poll timeout when doing group commit (Matteo Merli via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/master 6622b46d4 - fe6259c7e
BOOKKEEPER-850: Use nanoseconds to calculate poll timeout when doing group
commit (Matteo Merli via sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/fe6259c7
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/fe6259c7
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/fe6259c7
Branch: refs/heads/master
Commit: fe6259c7eade644c8d2a5e96aba61c1792d64843
Parents: 6622b46
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 00:23:48 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 00:23:48 2015 -0700
--
CHANGES.txt | 2 ++
.../org/apache/bookkeeper/bookie/Journal.java | 20 +++-
2 files changed, 13 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/fe6259c7/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 308a0a4..c36151d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -69,6 +69,8 @@ Trunk (unreleased changes)
BOOKKEEPER-849: Collect stats with sub-milliseconds precision (Matteo
Merli via sijie)
+ BOOKKEEPER-850: Use nanoseconds to calculate poll timeout when doing
group commit (Matteo Merli via sijie)
+
bookkeeper-client:
BOOKKEEPER-810: Allow to configure TCP connect timeout (Charles Xie
via sijie)
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/fe6259c7/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index cc61aa5..48e5f55 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -494,7 +494,7 @@ class Journal extends BookieCriticalThread implements
CheckpointSource {
final ServerConfiguration conf;
final ForceWriteThread forceWriteThread;
// Time after which we will stop grouping and issue the flush
-private final long maxGroupWaitInMSec;
+private final long maxGroupWaitInNanos;
// Threshold after which we flush any buffered journal entries
private final long bufferedEntriesThreshold;
// Threshold after which we flush any buffered journal writes
@@ -546,7 +546,7 @@ class Journal extends BookieCriticalThread implements
CheckpointSource {
this.journalWriteBufferSize = conf.getJournalWriteBufferSizeKB() * KB;
this.maxBackupJournals = conf.getMaxBackupJournals();
this.forceWriteThread = new ForceWriteThread(this,
conf.getJournalAdaptiveGroupWrites());
-this.maxGroupWaitInMSec = conf.getJournalMaxGroupWaitMSec();
+this.maxGroupWaitInNanos =
TimeUnit.MILLISECONDS.toNanos(conf.getJournalMaxGroupWaitMSec());
this.bufferedWritesThreshold =
conf.getJournalBufferedWritesThreshold();
this.bufferedEntriesThreshold =
conf.getJournalBufferedEntriesThreshold();
this.cbThreadPool =
Executors.newFixedThreadPool(conf.getNumJournalCallbackThreads(),
@@ -554,7 +554,7 @@ class Journal extends BookieCriticalThread implements
CheckpointSource {
// Unless there is a cap on the max wait (which requires group force
writes)
// we cannot skip flushing for queue empty
-this.flushWhenQueueEmpty = maxGroupWaitInMSec = 0 ||
conf.getJournalFlushWhenQueueEmpty();
+this.flushWhenQueueEmpty = maxGroupWaitInNanos = 0 ||
conf.getJournalFlushWhenQueueEmpty();
this.removePagesFromCache = conf.getJournalRemovePagesFromCache();
// read last log mark
@@ -822,17 +822,19 @@ class Journal extends BookieCriticalThread implements
CheckpointSource {
if (toFlush.isEmpty()) {
qe = queue.take();
} else {
-long pollWaitTime = maxGroupWaitInMSec -
MathUtils.elapsedMSec(toFlush.getFirst().enqueueTime);
-if (flushWhenQueueEmpty || pollWaitTime 0) {
-pollWaitTime = 0;
+long pollWaitTimeNanos = maxGroupWaitInNanos -
MathUtils.elapsedNanos(toFlush.get(0).enqueueTime);
+if (flushWhenQueueEmpty || pollWaitTimeNanos 0) {
+pollWaitTimeNanos = 0;
}
-qe = queue.poll(pollWaitTime, TimeUnit.MILLISECONDS);
+qe = queue.poll(pollWaitTimeNanos,
TimeUnit.NANOSECONDS);
bookkeeper git commit: BOOKKEEPER-849: Collect stats with sub-milliseconds precision (Matteo Merli via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/master 638b33256 - 6622b46d4
BOOKKEEPER-849: Collect stats with sub-milliseconds precision (Matteo Merli via
sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/6622b46d
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/6622b46d
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/6622b46d
Branch: refs/heads/master
Commit: 6622b46d4979c92cd9c3f450e3832d63513c6a22
Parents: 638b332
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 00:17:45 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 00:17:45 2015 -0700
--
CHANGES.txt | 2 ++
.../bookie/BookKeeperServerStats.java | 1 +
.../org/apache/bookkeeper/bookie/Bookie.java| 34 +++-
.../apache/bookkeeper/bookie/EntryMemTable.java | 18 +--
.../bookie/InterleavedLedgerStorage.java| 11 ---
.../org/apache/bookkeeper/bookie/Journal.java | 13
.../bookkeeper/client/LedgerCreateOp.java | 5 +--
.../bookkeeper/client/LedgerDeleteOp.java | 6 ++--
.../apache/bookkeeper/client/LedgerOpenOp.java | 6 ++--
.../apache/bookkeeper/client/PendingAddOp.java | 11 ---
.../apache/bookkeeper/client/PendingReadOp.java | 10 +++---
.../bookkeeper/proto/PacketProcessorBase.java | 6 ++--
.../bookkeeper/proto/PacketProcessorBaseV3.java | 6 ++--
.../proto/PerChannelBookieClient.java | 23 ++---
.../bookkeeper/proto/ReadEntryProcessor.java| 6 ++--
.../bookkeeper/proto/ReadEntryProcessorV3.java | 6 ++--
.../bookkeeper/proto/WriteEntryProcessor.java | 10 --
.../bookkeeper/proto/WriteEntryProcessorV3.java | 7 ++--
.../replication/ReplicationWorker.java | 6 ++--
.../org/apache/bookkeeper/util/MathUtils.java | 11 +++
.../bookkeeper/stats/CodahaleOpStatsLogger.java | 19 ---
.../bookkeeper/stats/CodahaleOpStatsTest.java | 2 +-
.../twitter/ostrich/OpStatsLoggerImpl.java | 22 ++---
.../twitter/science/OpStatsLoggerImpl.java | 16 ++---
.../bookkeeper/stats/NullStatsLogger.java | 16 +++--
.../apache/bookkeeper/stats/OpStatsLogger.java | 27
26 files changed, 200 insertions(+), 100 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/6622b46d/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 08c2705..308a0a4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -67,6 +67,8 @@ Trunk (unreleased changes)
BOOKKEEPER-844: Add more metrics about latency and bytes characteristics
on bookie operations (Tong Yu via sijie)
+ BOOKKEEPER-849: Collect stats with sub-milliseconds precision (Matteo
Merli via sijie)
+
bookkeeper-client:
BOOKKEEPER-810: Allow to configure TCP connect timeout (Charles Xie
via sijie)
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/6622b46d/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
index 9036182..c7919f7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
@@ -44,6 +44,7 @@ public interface BookKeeperServerStats {
public final static String BOOKIE_READ_ENTRY = BOOKIE_READ_ENTRY;
// Journal Stats
+public final static String JOURNAL_SCOPE = journal;
public final static String JOURNAL_ADD_ENTRY = JOURNAL_ADD_ENTRY;
public final static String JOURNAL_MEM_ADD_ENTRY = JOURNAL_MEM_ADD_ENTRY;
public final static String JOURNAL_PREALLOCATION = JOURNAL_PREALLOCATION;
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/6622b46d/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
index 8b40853..3078ff1 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
@@ -88,6 +88,7 @@ import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_BYTES;
import static
bookkeeper git commit: BOOKKEEPER-695: Some entry logs are not removed from the bookie storage (Matteo Merli via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/master 70da24d50 - b6dd50534
BOOKKEEPER-695: Some entry logs are not removed from the bookie storage (Matteo
Merli via sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/b6dd5053
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/b6dd5053
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/b6dd5053
Branch: refs/heads/master
Commit: b6dd505342051f267ca6af5fba1b70637c1e3fe0
Parents: 70da24d
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 00:44:52 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 00:44:52 2015 -0700
--
CHANGES.txt | 4
.../org/apache/bookkeeper/bookie/EntryLogger.java | 16
.../org/apache/bookkeeper/bookie/EntryLogTest.java | 6 +-
3 files changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/b6dd5053/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index c9842d1..e55b2d6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -79,6 +79,10 @@ Trunk (unreleased changes)
BOOKKEEPER-848: Use volatile for lastAddConfirmed (Matteo Merli via
sijie)
+ bookkeeper-server:
+
+BOOKKEEPER-695: Some entry logs are not removed from the bookie
storage (Matteo Merli via sijie)
+
Release 4.3.0 - 2014-10-03
Non-backward compatible changes:
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/b6dd5053/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index 0e052b5..725d5bf 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -782,20 +782,19 @@ public class EntryLogger {
break;
}
if (readFromLogChannel(entryLogId, bc, sizeBuff, pos) !=
sizeBuff.capacity()) {
-throw new IOException(Short read for entry size from entrylog
+ entryLogId);
+LOG.warn(Short read for entry size from entrylog {},
entryLogId);
+return;
}
long offset = pos;
pos += 4;
sizeBuff.flip();
int entrySize = sizeBuff.getInt();
-if (entrySize MB) {
-LOG.warn(Found large size entry of + entrySize + at
location + pos + in
-+ entryLogId);
-}
+
sizeBuff.clear();
// try to read ledger id first
if (readFromLogChannel(entryLogId, bc, lidBuff, pos) !=
lidBuff.capacity()) {
-throw new IOException(Short read for ledger id from entrylog
+ entryLogId);
+LOG.warn(Short read for ledger id from entrylog {},
entryLogId);
+return;
}
lidBuff.flip();
long lid = lidBuff.getLong();
@@ -810,8 +809,9 @@ public class EntryLogger {
ByteBuffer buff = ByteBuffer.wrap(data);
int rc = readFromLogChannel(entryLogId, bc, buff, pos);
if (rc != data.length) {
-throw new IOException(Short read for ledger entry from
entryLog + entryLogId
-+ @ + pos + ( + rc + != +
data.length + ));
+LOG.warn(Short read for ledger entry from entryLog {}@{} ({}
!= {}), new Object[] { entryLogId, pos,
+rc, data.length });
+return;
}
buff.flip();
// process the entry
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/b6dd5053/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
--
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
index 488f4bf..6b0ecd8 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
@@ -89,11 +89,7 @@ public class EntryLogTest {
EntryLogMetadata meta = new EntryLogMetadata(0L);
ExtractionScanner scanner = new ExtractionScanner(meta);
-try {
-logger.scanEntryLog(0L, scanner);
-fail(Should not reach here!);
-
bookkeeper git commit: BOOKKEEPER-846: TestLedgerChecker times out (rakeshr via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/branch-4.3 49395cf59 - 49261795f
BOOKKEEPER-846: TestLedgerChecker times out (rakeshr via sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/49261795
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/49261795
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/49261795
Branch: refs/heads/branch-4.3
Commit: 49261795f1a7b1b690b4fd81b5e01df8cf1ff1a6
Parents: 49395cf
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 01:14:25 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 01:14:25 2015 -0700
--
CHANGES.txt | 2 ++
.../bookkeeper/client/TestLedgerChecker.java| 35 ++--
2 files changed, 26 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/49261795/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index ecaf17c..9edfa96 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -26,6 +26,8 @@ Release 4.3.1 - unreleased
BOOKKEEPER-834: test case error in test class TestDiskChecker (zhaijia
via sijie)
+ BOOKKEEPER-846: TestLedgerChecker times out (rakeshr via sijie)
+
IMPROVEMENTS:
BOOKKEEPER-800: Expose whether a ledger is closed or not (ivank)
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/49261795/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
--
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
index eb61c21..0316b37 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
@@ -120,7 +120,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
// /we don't have any missed entries. Quorum satisfied//
// /So, there should not be any missing replicas.///
// /
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldNotGetTheFragmentIfThereIsNoMissedEntry()
throws Exception {
@@ -164,7 +164,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should give two fragments when 2 bookies failed
* in same ensemble when ensemble = 3, quorum = 2
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldGetTwoFrgamentsIfTwoBookiesFailedInSameEnsemble()
throws Exception {
@@ -198,7 +198,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should not get any underReplicated fragments,
if
* corresponding ledger does not exists.
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldNotGetAnyFragmentIfNoLedgerPresent()
throws Exception {
@@ -212,7 +212,20 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
startNewBookie();
lh.addEntry(TEST_LEDGER_ENTRY_DATA);
bkc.deleteLedger(lh.getId());
-
+LOG.info(Waiting to see ledger id {} deletion, lh.getId());
+int retries = 40;
+boolean noSuchLedger = false;
+while (retries 0) {
+try {
+lh.readEntries(0, 0);
+} catch (BKException.BKNoSuchLedgerExistsException bkn) {
+noSuchLedger = true;
+break;
+}
+retries--;
+Thread.sleep(500);
+}
+assertEquals(Ledger exists, true, noSuchLedger);
SetLedgerFragment result = getUnderReplicatedFragments(lh);
assertNotNull(Result shouldn't be null, result);
@@ -224,7 +237,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should get failed ensemble number of fragments
* if ensemble bookie failures on next entry
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void
testShouldGetFailedEnsembleNumberOfFgmntsIfEnsembleBookiesFailedOnNextWrite()
throws Exception {
@@ -258,7 +271,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should not get any fragments as underReplicated
* if Ledger itself is empty
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldNotGetAnyFragmentWithEmptyLedger() throws Exception {
bookkeeper git commit: BOOKKEEPER-847: ArrayIndexOutOfBoundsException in LedgerFragmentReplicator::updateEnsembleInfo (zhaijia via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/master fe6259c7e - b24cd7fd8
BOOKKEEPER-847: ArrayIndexOutOfBoundsException in
LedgerFragmentReplicator::updateEnsembleInfo (zhaijia via sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/b24cd7fd
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/b24cd7fd
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/b24cd7fd
Branch: refs/heads/master
Commit: b24cd7fd85bb3235036e660ff3a295dc939705e6
Parents: fe6259c
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 00:35:22 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 00:35:22 2015 -0700
--
CHANGES.txt | 2 ++
.../client/LedgerFragmentReplicator.java| 20
2 files changed, 18 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/b24cd7fd/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index c36151d..9ce143b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -34,6 +34,8 @@ Trunk (unreleased changes)
BOOKKEEPER-840: Deadlock on flushLock on compaction (sijie)
+ BOOKKEEPER-847: ArrayIndexOutOfBoundsException in
LedgerFragmentReplicator::updateEnsembleInfo (zhaijia via sijie)
+
IMPROVEMENTS:
BOOKKEEPER-800: Expose whether a ledger is closed or not (ivank)
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/b24cd7fd/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
index 4501524..2078245 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
@@ -345,10 +345,22 @@ public class LedgerFragmentReplicator {
ArrayListBookieSocketAddress ensemble = lh.getLedgerMetadata()
.getEnsembles().get(fragmentStartId);
int deadBookieIndex = ensemble.indexOf(oldBookie);
-ensemble.remove(deadBookieIndex);
-ensemble.add(deadBookieIndex, newBookie);
-lh.writeLedgerConfig(new UpdateEnsembleCb(ensembleUpdatedCb,
-fragmentStartId, lh, oldBookie, newBookie));
+
+/*
+ * An update to the ensemble info might happen after re-reading ledger
metadata.
+ * Such an update might reflect a change to the ensemble membership
such that
+ * it might not be necessary to replace the bookie.
+ */
+if (deadBookieIndex = 0) {
+ensemble.remove(deadBookieIndex);
+ensemble.add(deadBookieIndex, newBookie);
+lh.writeLedgerConfig(new UpdateEnsembleCb(ensembleUpdatedCb,
+fragmentStartId, lh, oldBookie, newBookie));
+}
+else {
+LOG.warn(Bookie {} doesn't exist in ensemble {} anymore.,
oldBookie, ensemble);
+
ensembleUpdatedCb.processResult(BKException.Code.UnexpectedConditionException,
null, null);
+}
}
/**
bookkeeper git commit: BOOKKEEPER-836: disable compaction when disk becomes full, otherwise compaction will fill up disk quickly (zhaijia via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/master b6dd50534 - e0d331781
BOOKKEEPER-836: disable compaction when disk becomes full, otherwise compaction
will fill up disk quickly (zhaijia via sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/e0d33178
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/e0d33178
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/e0d33178
Branch: refs/heads/master
Commit: e0d331781ec7a0e415bc6f16c38686aab34cb0c5
Parents: b6dd505
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 00:55:03 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 00:55:03 2015 -0700
--
CHANGES.txt | 2 +
.../bookie/GarbageCollectorThread.java | 53 ---
.../bookie/InterleavedLedgerStorage.java| 40 +++---
.../bookkeeper/conf/ServerConfiguration.java| 27 ++
.../bookkeeper/bookie/CompactionTest.java | 55
5 files changed, 164 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/e0d33178/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index e55b2d6..686b867 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -83,6 +83,8 @@ Trunk (unreleased changes)
BOOKKEEPER-695: Some entry logs are not removed from the bookie
storage (Matteo Merli via sijie)
+BOOKKEEPER-836: disable compaction when disk becomes full, otherwise
compaction will fill up disk quickly (zhaijia via sijie)
+
Release 4.3.0 - 2014-10-03
Non-backward compatible changes:
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/e0d33178/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index 299fb3e..1ca43e0 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -66,6 +66,8 @@ public class GarbageCollectorThread extends BookieThread {
final double majorCompactionThreshold;
final long majorCompactionInterval;
+final boolean isForceGCAllowWhenNoSpace;
+
long lastMinorCompactionTime;
long lastMajorCompactionTime;
@@ -93,6 +95,10 @@ public class GarbageCollectorThread extends BookieThread {
// Boolean to trigger a forced GC.
final AtomicBoolean forceGarbageCollection = new AtomicBoolean(false);
+// Boolean to disable major compaction, when disk is almost full
+final AtomicBoolean suspendMajorCompaction = new AtomicBoolean(false);
+// Boolean to disable minor compaction, when disk is full
+final AtomicBoolean suspendMinorCompaction = new AtomicBoolean(false);
final GarbageCollector garbageCollector;
final GarbageCleaner garbageCleaner;
@@ -264,6 +270,7 @@ public class GarbageCollectorThread extends BookieThread {
minorCompactionInterval = conf.getMinorCompactionInterval() * SECOND;
majorCompactionThreshold = conf.getMajorCompactionThreshold();
majorCompactionInterval = conf.getMajorCompactionInterval() * SECOND;
+isForceGCAllowWhenNoSpace = conf.getIsForceGCAllowWhenNoSpace();
if (minorCompactionInterval 0 minorCompactionThreshold 0) {
if (minorCompactionThreshold 1.0f) {
@@ -321,6 +328,30 @@ public class GarbageCollectorThread extends BookieThread {
}
}
+public void suspendMajorGC() {
+if (suspendMajorCompaction.compareAndSet(false, true)) {
+LOG.info(Suspend Major Compaction triggered by thread: {},
Thread.currentThread().getName());
+}
+}
+
+public void resumeMajorGC() {
+if (suspendMajorCompaction.compareAndSet(true, false)) {
+LOG.info({} Major Compaction back to normal since bookie has
enough space now., Thread.currentThread().getName());
+}
+}
+
+public void suspendMinorGC() {
+if (suspendMinorCompaction.compareAndSet(false, true)) {
+LOG.info(Suspend Minor Compaction triggered by thread: {},
Thread.currentThread().getName());
+}
+}
+
+public void resumeMinorGC() {
+if (suspendMinorCompaction.compareAndSet(true, false)) {
+LOG.info({} Minor Compaction back to normal since bookie has
enough space now., Thread.currentThread().getName());
+}
+}
+
@Override
public void
bookkeeper git commit: BOOKKEEPER-833: EntryLogId and EntryLogLimit should not be larger than Integer.MAX_VALUE (sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/branch-4.3 9cbbd659c - 49395cf59
BOOKKEEPER-833: EntryLogId and EntryLogLimit should not be larger than
Integer.MAX_VALUE (sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/49395cf5
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/49395cf5
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/49395cf5
Branch: refs/heads/branch-4.3
Commit: 49395cf5919c03af0f6951055ae1bfd7aef09d04
Parents: 9cbbd65
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 01:08:20 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 01:08:20 2015 -0700
--
CHANGES.txt | 4 +++
.../apache/bookkeeper/bookie/EntryLogger.java | 34 +---
.../bookkeeper/bookie/SortedLedgerStorage.java | 11 ++-
.../bookkeeper/conf/ServerConfiguration.java| 5 +++
.../bookkeeper/util/BookKeeperConstants.java| 7
5 files changed, 48 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/49395cf5/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index d3b1e88..ecaf17c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -44,6 +44,10 @@ Release 4.3.1 - unreleased
BOOKKEEPER-810: Allow to configure TCP connect timeout (Charles Xie
via sijie)
+ bookkeeper-server:
+
+BOOKKEEPER-833: EntryLogId and EntryLogLimit should not be larger than
Integer.MAX_VALUE (sijie)
+
Release 4.3.0 - 2014-10-03
Non-backward compatible changes:
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/49395cf5/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
--
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index 0e052b5..91ac8b4 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -22,6 +22,7 @@
package org.apache.bookkeeper.bookie;
import static com.google.common.base.Charsets.UTF_8;
+import static
org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -167,7 +168,7 @@ public class EntryLogger {
addListener(listener);
}
// log size limit
-this.logSizeLimit = conf.getEntryLogSizeLimit();
+this.logSizeLimit = Math.min(conf.getEntryLogSizeLimit(),
MAX_LOG_SIZE_LIMIT);
this.entryLogPreAllocationEnabled =
conf.isEntryLogFilePreAllocationEnabled();
// Initialize the entry log header buffer. This cannot be a static
object
@@ -439,7 +440,12 @@ public class EntryLogger {
// It would better not to overwrite existing entry log files
File newLogFile = null;
do {
-String logFileName = Long.toHexString(++preallocatedLogId) +
.log;
+if (preallocatedLogId = Integer.MAX_VALUE) {
+preallocatedLogId = 0;
+} else {
+++preallocatedLogId;
+}
+String logFileName = Long.toHexString(preallocatedLogId) +
.log;
for (File dir : list) {
newLogFile = new File(dir, logFileName);
currentDir = dir;
@@ -622,15 +628,16 @@ public class EntryLogger {
}
synchronized long addEntry(long ledger, ByteBuffer entry, boolean rollLog)
throws IOException {
-if (rollLog) {
-// Create new log if logSizeLimit reached or current disk is full
-boolean createNewLog = shouldCreateNewEntryLog.get();
-if (createNewLog || reachEntryLogLimit(entry.remaining() + 4)) {
-createNewLog();
-// Reset the flag
-if (createNewLog) {
-shouldCreateNewEntryLog.set(false);
-}
+int entrySize = entry.remaining() + 4;
+boolean reachEntryLogLimit =
+rollLog ? reachEntryLogLimit(entrySize) :
readEntryLogHardLimit(entrySize);
+// Create new log if logSizeLimit reached or current disk is full
+boolean createNewLog = shouldCreateNewEntryLog.get();
+if (createNewLog || reachEntryLogLimit) {
+createNewLog();
+// Reset the flag
+if (createNewLog) {
+shouldCreateNewEntryLog.set(false);
}
}
ByteBuffer buff = ByteBuffer.allocate(4);
@@ -651,6 +658,10 @@
bookkeeper git commit: BOOKKEEPER-846: TestLedgerChecker times out (rakeshr via sijie)
Repository: bookkeeper
Updated Branches:
refs/heads/master da1a2fa6b - 26b17cba5
BOOKKEEPER-846: TestLedgerChecker times out (rakeshr via sijie)
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/26b17cba
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/26b17cba
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/26b17cba
Branch: refs/heads/master
Commit: 26b17cba50aded931afb2e66d549fbeb8f4f1d39
Parents: da1a2fa
Author: Sijie Guo si...@apache.org
Authored: Tue Apr 21 01:19:24 2015 -0700
Committer: Sijie Guo si...@apache.org
Committed: Tue Apr 21 01:19:24 2015 -0700
--
CHANGES.txt | 2 ++
.../bookkeeper/client/TestLedgerChecker.java| 35 ++--
2 files changed, 26 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b17cba/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 060d814..d3976d9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -36,6 +36,8 @@ Trunk (unreleased changes)
BOOKKEEPER-847: ArrayIndexOutOfBoundsException in
LedgerFragmentReplicator::updateEnsembleInfo (zhaijia via sijie)
+ BOOKKEEPER-846: TestLedgerChecker times out (rakeshr via sijie)
+
IMPROVEMENTS:
BOOKKEEPER-800: Expose whether a ledger is closed or not (ivank)
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b17cba/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
--
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
index eb61c21..0316b37 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestLedgerChecker.java
@@ -120,7 +120,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
// /we don't have any missed entries. Quorum satisfied//
// /So, there should not be any missing replicas.///
// /
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldNotGetTheFragmentIfThereIsNoMissedEntry()
throws Exception {
@@ -164,7 +164,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should give two fragments when 2 bookies failed
* in same ensemble when ensemble = 3, quorum = 2
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldGetTwoFrgamentsIfTwoBookiesFailedInSameEnsemble()
throws Exception {
@@ -198,7 +198,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should not get any underReplicated fragments,
if
* corresponding ledger does not exists.
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldNotGetAnyFragmentIfNoLedgerPresent()
throws Exception {
@@ -212,7 +212,20 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
startNewBookie();
lh.addEntry(TEST_LEDGER_ENTRY_DATA);
bkc.deleteLedger(lh.getId());
-
+LOG.info(Waiting to see ledger id {} deletion, lh.getId());
+int retries = 40;
+boolean noSuchLedger = false;
+while (retries 0) {
+try {
+lh.readEntries(0, 0);
+} catch (BKException.BKNoSuchLedgerExistsException bkn) {
+noSuchLedger = true;
+break;
+}
+retries--;
+Thread.sleep(500);
+}
+assertEquals(Ledger exists, true, noSuchLedger);
SetLedgerFragment result = getUnderReplicatedFragments(lh);
assertNotNull(Result shouldn't be null, result);
@@ -224,7 +237,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should get failed ensemble number of fragments
* if ensemble bookie failures on next entry
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void
testShouldGetFailedEnsembleNumberOfFgmntsIfEnsembleBookiesFailedOnNextWrite()
throws Exception {
@@ -258,7 +271,7 @@ public class TestLedgerChecker extends
BookKeeperClusterTestCase {
* Tests that LedgerChecker should not get any fragments as underReplicated
* if Ledger itself is empty
*/
-@Test(timeout = 3000)
+@Test(timeout = 6)
public void testShouldNotGetAnyFragmentWithEmptyLedger()
