This is an automated email from the ASF dual-hosted git repository. lhotari pushed a commit to branch branch-4.16 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit c39837965872572d45293578decc9081fafc04bb Author: fengyubiao <[email protected]> AuthorDate: Wed Sep 17 12:29:19 2025 +0800 [fix]BK stays at read_only state even if the disk is empty (#4640) * [fix]BK stay at read_only state even if the disk is empty * correct the javadoc * improve test (cherry picked from commit bde9ff2503ae71c4ef03b212577a1a8be673123a) --- .../bookkeeper/bookie/BookieStateManager.java | 21 +-- .../org/apache/bookkeeper/bookie/BookieStatus.java | 19 ++- .../bookkeeper/bookie/LedgerDirsMonitor.java | 17 ++- .../org/apache/bookkeeper/bookie/StateManager.java | 12 +- .../http/service/BookieStateReadOnlyService.java | 4 +- .../bookie/BookieInitializationTest.java | 153 +++++++++++++++++++-- .../apache/bookkeeper/bookie/LedgerCacheTest.java | 2 +- .../replication/AuditorLedgerCheckerTest.java | 4 +- 8 files changed, 202 insertions(+), 30 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java index 620a1c2aaa..c50d51a96a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java @@ -165,7 +165,7 @@ public class BookieStateManager implements StateManager { @Override public void initState(){ if (forceReadOnly.get()) { - this.bookieStatus.setToReadOnlyMode(); + this.bookieStatus.setToReadOnlyMode(false); } else if (conf.isPersistBookieStatusEnabled()) { this.bookieStatus.readFromDirectories(statusDirs); } @@ -256,22 +256,22 @@ public class BookieStateManager implements StateManager { } @Override - public Future<Void> transitionToWritableMode() { + public Future<Void> transitionToWritableMode(boolean isManuallyModify) { return stateService.submit(new Callable<Void>() { @Override public Void call() throws Exception{ - doTransitionToWritableMode(); + doTransitionToWritableMode(isManuallyModify); return null; } }); } @Override - public Future<Void> transitionToReadOnlyMode() { + public Future<Void> transitionToReadOnlyMode(boolean isManuallyModify) { return stateService.submit(new Callable<Void>() { @Override public Void call() throws Exception{ - doTransitionToReadOnlyMode(); + doTransitionToReadOnlyMode(isManuallyModify); return null; } }); @@ -298,10 +298,15 @@ public class BookieStateManager implements StateManager { } @VisibleForTesting - public void doTransitionToWritableMode() { + public void doTransitionToWritableMode(boolean isManuallyModify) { if (shuttingdown || forceReadOnly.get()) { return; } + if (!isManuallyModify && bookieStatus.isInReadOnlyMode() && bookieStatus.isManuallyModifiedToReadOnly()) { + LOG.info("Skip to transition Bookie to Writable mode automatically because it is manually set to read-only" + + " mode, which can only be changed manually."); + return; + } if (!bookieStatus.setToWritableMode()) { // do nothing if already in writable mode @@ -334,11 +339,11 @@ public class BookieStateManager implements StateManager { } @VisibleForTesting - public void doTransitionToReadOnlyMode() { + public void doTransitionToReadOnlyMode(boolean isManuallyModify) { if (shuttingdown) { return; } - if (!bookieStatus.setToReadOnlyMode()) { + if (!bookieStatus.setToReadOnlyMode(isManuallyModify)) { return; } if (!conf.isReadOnlyModeEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java index 7b2d0aad99..ccbba36b53 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.List; +import lombok.Getter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,8 @@ public class BookieStatus { private int layoutVersion; private long lastUpdateTime; private volatile BookieMode bookieMode; + @Getter + private boolean isManuallyModifiedToReadOnly; BookieStatus() { this.bookieMode = BookieMode.READ_WRITE; @@ -72,6 +75,7 @@ public class BookieStatus { if (!bookieMode.equals(BookieMode.READ_WRITE)) { bookieMode = BookieMode.READ_WRITE; this.lastUpdateTime = System.currentTimeMillis(); + this.isManuallyModifiedToReadOnly = false; return true; } return false; @@ -81,10 +85,11 @@ public class BookieStatus { return bookieMode.equals(BookieMode.READ_ONLY); } - synchronized boolean setToReadOnlyMode() { + synchronized boolean setToReadOnlyMode(boolean isManuallyModify) { if (!bookieMode.equals(BookieMode.READ_ONLY)) { bookieMode = BookieMode.READ_ONLY; this.lastUpdateTime = System.currentTimeMillis(); + this.isManuallyModifiedToReadOnly = isManuallyModify; return true; } return false; @@ -147,6 +152,7 @@ public class BookieStatus { this.lastUpdateTime = status.lastUpdateTime; this.layoutVersion = status.layoutVersion; this.bookieMode = status.bookieMode; + this.isManuallyModifiedToReadOnly = status.isManuallyModifiedToReadOnly; success = true; } } @@ -216,6 +222,15 @@ public class BookieStatus { if (status.layoutVersion == 1 && parts.length == 3) { status.bookieMode = BookieMode.valueOf(parts[1]); status.lastUpdateTime = Long.parseLong(parts[2].trim()); + status.isManuallyModifiedToReadOnly = true; + return status; + } + // Since we should guarantee the compatibility of downgrade. We do not change the layoutVersion, otherwise, + // the string can not be parsed by the lower version. + if (status.layoutVersion == 1 && parts.length == 4) { + status.bookieMode = BookieMode.valueOf(parts[1]); + status.lastUpdateTime = Long.parseLong(parts[2].trim()); + status.isManuallyModifiedToReadOnly = Boolean.parseBoolean(parts[3].trim()); return status; } } @@ -231,6 +246,8 @@ public class BookieStatus { builder.append(getBookieMode()); builder.append(","); builder.append(System.currentTimeMillis()); + builder.append(","); + builder.append(isManuallyModifiedToReadOnly); builder.append("\n"); return builder.toString(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java index 7ad8ba1e0c..73e2ba1871 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java @@ -21,6 +21,7 @@ package org.apache.bookkeeper.bookie; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; @@ -54,6 +55,7 @@ class LedgerDirsMonitor { private final long minUsableSizeForHighPriorityWrites; private ScheduledExecutorService executor; private ScheduledFuture<?> checkTask; + private boolean isFirstLoopOfCheckTask = true; public LedgerDirsMonitor(final ServerConfiguration conf, final DiskChecker diskChecker, @@ -67,6 +69,10 @@ class LedgerDirsMonitor { } private void check(final LedgerDirsManager ldm) { + final boolean isFirstLoopOfCheckTaskLocalValue = this.isFirstLoopOfCheckTask; + if (isFirstLoopOfCheckTaskLocalValue) { + this.isFirstLoopOfCheckTask = false; + } final ConcurrentMap<File, Float> diskUsages = ldm.getDiskUsages(); boolean someDiskFulled = false; boolean highPriorityWritesAllowed = true; @@ -175,6 +181,14 @@ class LedgerDirsMonitor { } } + if (isFirstLoopOfCheckTaskLocalValue && ldm.getFullFilledLedgerDirs().isEmpty()) { + // notify no disk full. + for (LedgerDirsListener listener : ldm.getListeners()) { + listener.allDisksWritable(); + } + return; + } + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { if (someDiskFulled && !ldm.getFullFilledLedgerDirs().isEmpty()) { // notify any disk full. @@ -192,7 +206,8 @@ class LedgerDirsMonitor { } } - private void check() { + @VisibleForTesting + void check() { dirsManagers.forEach(this::check); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java index 7ed3f0b657..9ef1f384e7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java @@ -100,12 +100,20 @@ public interface StateManager extends AutoCloseable { /** * Change the state of bookie to Writable mode. */ - Future<Void> transitionToWritableMode(); + Future<Void> transitionToWritableMode(boolean isManuallyModify); + + default Future<Void> transitionToWritableMode() { + return transitionToWritableMode(false); + } /** * Change the state of bookie to ReadOnly mode. */ - Future<Void> transitionToReadOnlyMode(); + Future<Void> transitionToReadOnlyMode(boolean isManuallyModify); + + default Future<Void> transitionToReadOnlyMode() { + return transitionToReadOnlyMode(false); + } /** * ShutdownHandler used to shutdown bookie. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java index d32074e2bb..73f57c1959 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java @@ -58,14 +58,14 @@ public class BookieStateReadOnlyService implements HttpEndpointService { response.setBody("Bookie is in forceReadOnly mode, cannot transit to writable mode"); return response; } - stateManager.transitionToWritableMode().get(); + stateManager.transitionToWritableMode(true).get(); } else if (!stateManager.isReadOnly() && inState.isReadOnly()) { if (!stateManager.isReadOnlyModeEnabled()) { response.setCode(HttpServer.StatusCode.BAD_REQUEST); response.setBody("Bookie is disabled ReadOnly mode, cannot transit to readOnly mode"); return response; } - stateManager.transitionToReadOnlyMode().get(); + stateManager.transitionToReadOnlyMode(true).get(); } } else if (!HttpServer.Method.GET.equals(request.getMethod())) { response.setCode(HttpServer.StatusCode.NOT_FOUND); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java index 5188eb816b..bd0a12ca6c 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java @@ -77,6 +77,7 @@ import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.component.ComponentStarter; import org.apache.bookkeeper.common.component.Lifecycle; import org.apache.bookkeeper.common.component.LifecycleComponent; +import org.apache.bookkeeper.common.util.JsonUtil; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; @@ -84,7 +85,10 @@ import org.apache.bookkeeper.discover.BookieServiceInfo; import org.apache.bookkeeper.discover.BookieServiceInfo.Endpoint; import org.apache.bookkeeper.discover.RegistrationManager; import org.apache.bookkeeper.http.HttpRouter; +import org.apache.bookkeeper.http.HttpServer; import org.apache.bookkeeper.http.HttpServerLoader; +import org.apache.bookkeeper.http.service.HttpServiceRequest; +import org.apache.bookkeeper.http.service.HttpServiceResponse; import org.apache.bookkeeper.meta.MetadataBookieDriver; import org.apache.bookkeeper.meta.exceptions.MetadataException; import org.apache.bookkeeper.meta.zk.ZKMetadataBookieDriver; @@ -96,6 +100,7 @@ import org.apache.bookkeeper.replication.AutoRecoveryMain; import org.apache.bookkeeper.replication.ReplicationStats; import org.apache.bookkeeper.server.Main; import org.apache.bookkeeper.server.conf.BookieConfiguration; +import org.apache.bookkeeper.server.http.service.BookieStateReadOnlyService; import org.apache.bookkeeper.server.service.AutoRecoveryService; import org.apache.bookkeeper.server.service.BookieService; import org.apache.bookkeeper.stats.NullStatsLogger; @@ -107,8 +112,10 @@ import org.apache.bookkeeper.util.PortManager; import org.apache.bookkeeper.versioning.Version; import org.apache.bookkeeper.versioning.Versioned; import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.io.FileUtils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import org.awaitility.Awaitility; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -1445,30 +1452,150 @@ public class BookieInitializationTest extends BookKeeperClusterTestCase { .setPersistBookieStatusEnabled(true) .setMetadataServiceUri(metadataServiceUri); // start a new bookie - BookieServer bookieServer = new BookieServer( + BookieServer bookieServer1 = new BookieServer( conf, new TestBookieImpl(conf), NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, new MockUncleanShutdownDetection()); - bookieServer.start(); + bookieServer1.start(); // transition in to read only and persist the status on disk - Bookie bookie = (BookieImpl) bookieServer.getBookie(); - assertFalse(bookie.isReadOnly()); - bookie.getStateManager().transitionToReadOnlyMode().get(); - assertTrue(bookie.isReadOnly()); + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + assertFalse(bookie1.isReadOnly()); + bookie1.getStateManager().transitionToReadOnlyMode().get(); + assertTrue(bookie1.isReadOnly()); // corrupt status file - List<File> ledgerDirs = ((BookieImpl) bookie).getLedgerDirsManager().getAllLedgerDirs(); + List<File> ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME)); corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME)); // restart the bookie should be in read only mode - bookieServer.shutdown(); - bookieServer = new BookieServer( + bookieServer1.shutdown(); + BookieServer bookieServer2 = new BookieServer( conf, new TestBookieImpl(conf), NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, new MockUncleanShutdownDetection()); - bookieServer.start(); - bookie = bookieServer.getBookie(); - assertTrue(bookie.isReadOnly()); - bookieServer.shutdown(); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should switch to writable mode because the disk usage is fine. + bookie2.dirsMonitor.check(); + Awaitility.await().untilAsserted(() -> { + assertFalse(bookie2.isReadOnly()); + }); + bookieServer2.shutdown(); + } + + + private void setBookieToReadOnly(Bookie bookie, boolean readOnly) throws Exception { + BookieStateReadOnlyService.ReadOnlyState state = new BookieStateReadOnlyService.ReadOnlyState(); + state.setReadOnly(readOnly); + HttpServiceRequest request = new HttpServiceRequest(JsonUtil.toJson(state), HttpServer.Method.PUT, null); + BookieStateReadOnlyService service = new BookieStateReadOnlyService(bookie); + HttpServiceResponse response1 = service.handle(request); + assertEquals(HttpServer.StatusCode.OK.getValue(), response1.getStatusCode()); + } + + /** + * Verify: once the state is manually modified to read-only by Admin API, it should not be changed to writable + * by the monitor task. + * But it can be changed to read-only by monitor task if it was manually set to writable by Admin API. + */ + @Test(timeout = 1000 * 30) + public void testRetrieveBookieStatusAdminModifiedWhenStatusFileIsCorrupted() throws Exception { + File[] tmpLedgerDirs = new File[3]; + String[] filePath = new String[tmpLedgerDirs.length]; + for (int i = 0; i < tmpLedgerDirs.length; i++) { + tmpLedgerDirs[i] = tmpDirs.createNew("bookie", "test" + i); + filePath[i] = tmpLedgerDirs[i].getPath(); + } + final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setJournalDirName(filePath[0]) + .setLedgerDirNames(filePath) + .setReadOnlyModeEnabled(true) + .setPersistBookieStatusEnabled(true) + .setMetadataServiceUri(metadataServiceUri); + // start a new bookie + BookieServer bookieServer1 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer1.start(); + // transition in to read only and persist the status on disk + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + assertFalse(bookie1.isReadOnly()); + setBookieToReadOnly(bookie1, true); + assertTrue(bookie1.isReadOnly()); + // corrupt status file + List<File> ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); + corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME)); + corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME)); + // restart the bookie should be in read only mode + bookieServer1.shutdown(); + BookieServer bookieServer2 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should not switch to writable mode because the state was set to read-only + // by admin api manually. + bookie2.dirsMonitor.check(); + Thread.sleep(2000); + Awaitility.await().untilAsserted(() -> { + assertTrue(bookie2.isReadOnly()); + }); + // We can use Admin Api to change the state back to writable. + setBookieToReadOnly(bookie2, false); + assertFalse(bookie2.isReadOnly()); + // The state can be changed to read-only by monitor task if it was manually set to writable by Admin API. + bookie2.getStateManager().transitionToReadOnlyMode().get(); + assertTrue(bookie2.isReadOnly()); + bookieServer2.shutdown(); + } + + /** + * Verify: the newest version can read the old version payload of persisted bookie status. + * old payload: "1,READ_ONLY,1752809349613" + * new payload: "1,READ_ONLY,1752809349613,false" + */ + @Test(timeout = 30000) + public void testPersistBookieStatusCompatibility() throws Exception { + File[] tmpLedgerDirs = new File[1]; + String[] filePath = new String[1]; + tmpLedgerDirs[0] = tmpDirs.createNew("bookie", "test"); + filePath[0] = tmpLedgerDirs[0].getPath(); + final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setJournalDirName(filePath[0]) + .setLedgerDirNames(filePath) + .setReadOnlyModeEnabled(true) + .setPersistBookieStatusEnabled(true) + .setMetadataServiceUri(metadataServiceUri); + // start a new bookie + BookieServer bookieServer1 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer1.start(); + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + // restart the bookie should be in read only mode + bookieServer1.shutdown(); + // Rewrite file to the old version payload. + List<File> ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); + FileUtils.writeStringToFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME), + "1,READ_ONLY,1752809349613"); + BookieServer bookieServer2 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should not switch to writable mode because the state was set to read-only + // by admin api manually(the default value of previous version is true). + bookie2.dirsMonitor.check(); + Awaitility.await().untilAsserted(() -> { + assertTrue(bookie2.isReadOnly()); + }); + bookieServer2.shutdown(); } /** diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java index 4e8f06fe7f..9e2090d797 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java @@ -711,7 +711,7 @@ public class LedgerCacheTest { LOG.info("Started flushing mem table."); memTable.flush(FlushTestSortedLedgerStorage.this); } catch (IOException e) { - getStateManager().doTransitionToReadOnlyMode(); + getStateManager().doTransitionToReadOnlyMode(false); LOG.error("Exception thrown while flushing skip list cache.", e); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index 2e3e09012f..bbeb7e758f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -326,7 +326,7 @@ public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase { BookieServer bk = serverByIndex(bkIndex); bookieConf.setReadOnlyModeEnabled(true); - ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false); bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) .get(30, TimeUnit.SECONDS); @@ -361,7 +361,7 @@ public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase { BookieServer bk = serverByIndex(bkIndex); bookieConf.setReadOnlyModeEnabled(true); - ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false); bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) .get(30, TimeUnit.SECONDS);
