This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push: new 3ddc5db Change default ledger manager factory from `Flat` to `Hierachical` 3ddc5db is described below commit 3ddc5db190b049f181ce7ae29ca34cbc7a7b7eb3 Author: Sijie Guo <si...@apache.org> AuthorDate: Sat Feb 3 09:31:01 2018 -0800 Change default ledger manager factory from `Flat` to `Hierachical` Descriptions of the changes in this PR: the discussion can be found at : http://mail-archives.apache.org/mod_mbox/bookkeeper-dev/201802.mbox/%3CCAO2yDyYdZ8%3D4tViNX1uL0Z67KX78JPD2XBGd7ermjO%3DK%3DscvcQ%40mail.gmail.com%3E Author: Sijie Guo <si...@apache.org> Reviewers: Yiming Zang <yzang2...@gmail.com>, Jia Zhai <None>, Enrico Olivelli <eolive...@gmail.com> This closes #1115 from sijie/flip_default_ledger_manager --- bookkeeper-server/conf/bk_server.conf | 2 +- .../bookkeeper/meta/FlatLedgerManagerFactory.java | 3 +++ .../bookkeeper/meta/LedgerManagerFactory.java | 6 +++--- .../bookie/TestGcOverreplicatedLedger.java | 5 ++--- .../bookkeeper/client/TestWatchEnsembleChange.java | 4 ++-- .../bookkeeper/meta/LedgerManagerTestCase.java | 1 + .../apache/bookkeeper/meta/TestLedgerLayout.java | 6 +++--- .../apache/bookkeeper/meta/TestLedgerManager.java | 24 +++++++++++----------- .../apache/bookkeeper/meta/ZkLedgerLayoutTest.java | 14 ++++++------- deploy/kubernetes/gke/bookkeeper.statefulset.yml | 2 -- deploy/kubernetes/gke/bookkeeper.yaml | 2 -- site/_data/config/bk_server.yaml | 4 ++-- site/docs/latest/overview/releaseNotes.md | 6 ++++++ 13 files changed, 42 insertions(+), 37 deletions(-) diff --git a/bookkeeper-server/conf/bk_server.conf b/bookkeeper-server/conf/bk_server.conf index 7c82de2..464ff4a 100755 --- a/bookkeeper-server/conf/bk_server.conf +++ b/bookkeeper-server/conf/bk_server.conf @@ -402,7 +402,7 @@ ledgerDirectories=/tmp/bk-data ledgerManagerFactoryClass=org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory # @Drepcated - `ledgerManagerType` is deprecated in favor of using `ledgerManagerFactoryClass`. -# ledgerManagerType=flat +# ledgerManagerType=hierarchical # Root Zookeeper path to store ledger metadata # This parameter is used by zookeeper-based ledger manager as a root znode to diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/FlatLedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/FlatLedgerManagerFactory.java index 5e2ced5..12b5e18 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/FlatLedgerManagerFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/FlatLedgerManagerFactory.java @@ -30,7 +30,10 @@ import org.apache.zookeeper.data.ACL; /** * Flat Ledger Manager Factory. + * + * @deprecated since 4.7.0. The implementation will be still available but not recommended to use. */ +@Deprecated public class FlatLedgerManagerFactory extends AbstractZkLedgerManagerFactory { public static final String NAME = "flat"; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManagerFactory.java index 62ff092..e41554b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManagerFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManagerFactory.java @@ -106,6 +106,7 @@ public abstract class LedgerManagerFactory implements AutoCloseable { * @return new ledger manager factory * @throws IOException */ + @SuppressWarnings("deprecation") public static LedgerManagerFactory newLedgerManagerFactory( final AbstractConfiguration<?> conf, LayoutManager layoutManager) throws IOException, InterruptedException { @@ -146,7 +147,6 @@ public abstract class LedgerManagerFactory implements AutoCloseable { // handle pre V2 layout if (layout.getLayoutFormatVersion() <= V1) { // pre V2 layout we use type of ledger manager - @SuppressWarnings("deprecation") String lmType = conf.getLedgerManagerType(); if (lmType != null && !layout.getManagerFactoryClass().equals(lmType)) { throw new IOException("Configured layout " + lmType @@ -194,6 +194,7 @@ public abstract class LedgerManagerFactory implements AutoCloseable { * Creates the new layout and stores in zookeeper and returns the * LedgerManagerFactory instance. */ + @SuppressWarnings("deprecation") private static LedgerManagerFactory createNewLMFactory( final AbstractConfiguration conf, final LayoutManager layoutManager, Class<? extends LedgerManagerFactory> factoryClass) @@ -204,10 +205,9 @@ public abstract class LedgerManagerFactory implements AutoCloseable { // use default ledger manager factory if no one provided if (factoryClass == null) { // for backward compatibility, check manager type - @SuppressWarnings("deprecation") String lmType = conf.getLedgerManagerType(); if (lmType == null) { - factoryClass = FlatLedgerManagerFactory.class; + factoryClass = HierarchicalLedgerManagerFactory.class; } else { if (FlatLedgerManagerFactory.NAME.equals(lmType)) { factoryClass = FlatLedgerManagerFactory.class; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestGcOverreplicatedLedger.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestGcOverreplicatedLedger.java index fd2bf55..a422276 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestGcOverreplicatedLedger.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestGcOverreplicatedLedger.java @@ -33,14 +33,13 @@ import java.util.SortedMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; - import org.apache.bookkeeper.bookie.GarbageCollector.GarbageCleaner; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.LedgerMetadata; import org.apache.bookkeeper.conf.ServerConfiguration; -import org.apache.bookkeeper.meta.FlatLedgerManagerFactory; +import org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerManagerTestCase; import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; @@ -77,7 +76,7 @@ public class TestGcOverreplicatedLedger extends LedgerManagerTestCase { @Parameters public static Collection<Object[]> configs() { - return Arrays.asList(new Object[][] { { FlatLedgerManagerFactory.class } }); + return Arrays.asList(new Object[][] { { HierarchicalLedgerManagerFactory.class } }); } @Test diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestWatchEnsembleChange.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestWatchEnsembleChange.java index f79c27e..5d7be5d 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestWatchEnsembleChange.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestWatchEnsembleChange.java @@ -31,7 +31,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.discover.RegistrationManager; -import org.apache.bookkeeper.meta.FlatLedgerManagerFactory; import org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerIdGenerator; import org.apache.bookkeeper.meta.LedgerManager; @@ -71,10 +70,11 @@ public class TestWatchEnsembleChange extends BookKeeperClusterTestCase { baseConf.setLedgerManagerFactoryClass(lmFactoryCls); } + @SuppressWarnings("deprecation") @Parameters public static Collection<Object[]> configs() { return Arrays.asList(new Object[][] { - { FlatLedgerManagerFactory.class }, + { org.apache.bookkeeper.meta.FlatLedgerManagerFactory.class }, { HierarchicalLedgerManagerFactory.class }, { LongHierarchicalLedgerManagerFactory.class }, { MSLedgerManagerFactory.class }, diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/LedgerManagerTestCase.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/LedgerManagerTestCase.java index 4148575..78d3b08 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/LedgerManagerTestCase.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/LedgerManagerTestCase.java @@ -89,6 +89,7 @@ public abstract class LedgerManagerTestCase extends BookKeeperClusterTestCase { return ledgerIdGenerator; } + @SuppressWarnings("deprecation") @Parameters public static Collection<Object[]> configs() { return Arrays.asList(new Object[][] { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerLayout.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerLayout.java index 655713e..62315fe 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerLayout.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerLayout.java @@ -38,16 +38,16 @@ public class TestLedgerLayout { HierarchicalLedgerManagerFactory.class.getName(), 2); - private static final LedgerLayout flat1 = + private static final LedgerLayout longHierarchical = new LedgerLayout( - FlatLedgerManagerFactory.class.getName(), + LongHierarchicalLedgerManagerFactory.class.getName(), 1); @Test public void testEquals() { assertEquals(hierarchical1, hierarchical1); assertNotEquals(hierarchical1, hierarchical2); - assertNotEquals(hierarchical1, flat1); + assertNotEquals(hierarchical1, longHierarchical); } @Test diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerManager.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerManager.java index 744b9af..a466590 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerManager.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerManager.java @@ -88,11 +88,11 @@ public class TestLedgerManager extends BookKeeperClusterTestCase { conf, zkLayoutManager); assertTrue("Ledger manager is unexpected type", - (m instanceof FlatLedgerManagerFactory)); + (m instanceof HierarchicalLedgerManagerFactory)); m.close(); // mismatching conf - conf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class); + conf.setLedgerManagerFactoryClass(LongHierarchicalLedgerManagerFactory.class); try { LedgerManagerFactory.newLedgerManagerFactory(conf, zkLayoutManager); fail("Shouldn't reach here"); @@ -132,10 +132,10 @@ public class TestLedgerManager extends BookKeeperClusterTestCase { Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); conf.setZkLedgersRootPath(root0); // write v1 layout - writeLedgerLayout(root0, FlatLedgerManagerFactory.NAME, - FlatLedgerManagerFactory.CUR_VERSION, 1); + writeLedgerLayout(root0, HierarchicalLedgerManagerFactory.NAME, + HierarchicalLedgerManagerFactory.CUR_VERSION, 1); - conf.setLedgerManagerFactoryClass(FlatLedgerManagerFactory.class); + conf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class); ZkLayoutManager zkLayoutManager = new ZkLayoutManager( zkc, @@ -147,18 +147,18 @@ public class TestLedgerManager extends BookKeeperClusterTestCase { zkLayoutManager); assertTrue("Ledger manager is unexpected type", - (m instanceof FlatLedgerManagerFactory)); + (m instanceof HierarchicalLedgerManagerFactory)); m.close(); // v2 setting doesn't effect v1 conf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class); m = LedgerManagerFactory.newLedgerManagerFactory(conf, zkLayoutManager); assertTrue("Ledger manager is unexpected type", - (m instanceof FlatLedgerManagerFactory)); + (m instanceof HierarchicalLedgerManagerFactory)); m.close(); // mismatching conf - conf.setLedgerManagerType(HierarchicalLedgerManagerFactory.NAME); + conf.setLedgerManagerType(LongHierarchicalLedgerManagerFactory.NAME); try { LedgerManagerFactory.newLedgerManagerFactory(conf, zkLayoutManager); fail("Shouldn't reach here"); @@ -203,7 +203,7 @@ public class TestLedgerManager extends BookKeeperClusterTestCase { Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); conf.setZkLedgersRootPath(root1); - LedgerLayout layout1 = new LedgerLayout(FlatLedgerManagerFactory.class.getName(), + LedgerLayout layout1 = new LedgerLayout(HierarchicalLedgerManagerFactory.class.getName(), 0xdeadbeef); ZkLayoutManager zkLayoutManager1 = new ZkLayoutManager(zkc, root1, ZooDefs.Ids.OPEN_ACL_UNSAFE); zkLayoutManager1.storeLedgerLayout(layout1); @@ -279,7 +279,7 @@ public class TestLedgerManager extends BookKeeperClusterTestCase { List<CreateLMThread> threads = new ArrayList<CreateLMThread>(numThreads); for (int i = 0; i < numThreads; i++) { CreateLMThread t = new CreateLMThread(zkUtil.getZooKeeperConnectString(), - root0, FlatLedgerManagerFactory.class.getName(), barrier); + root0, HierarchicalLedgerManagerFactory.class.getName(), barrier); t.start(); threads.add(t); } @@ -309,14 +309,14 @@ public class TestLedgerManager extends BookKeeperClusterTestCase { List<CreateLMThread> threadsA = new ArrayList<CreateLMThread>(numThreadsEach); for (int i = 0; i < numThreadsEach; i++) { CreateLMThread t = new CreateLMThread(zkUtil.getZooKeeperConnectString(), - root0, FlatLedgerManagerFactory.class.getName(), barrier); + root0, HierarchicalLedgerManagerFactory.class.getName(), barrier); t.start(); threadsA.add(t); } List<CreateLMThread> threadsB = new ArrayList<CreateLMThread>(numThreadsEach); for (int i = 0; i < numThreadsEach; i++) { CreateLMThread t = new CreateLMThread(zkUtil.getZooKeeperConnectString(), - root0, HierarchicalLedgerManagerFactory.class.getName(), barrier); + root0, LongHierarchicalLedgerManagerFactory.class.getName(), barrier); t.start(); threadsB.add(t); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/ZkLedgerLayoutTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/ZkLedgerLayoutTest.java index cbf89ac..a72cce5 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/ZkLedgerLayoutTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/ZkLedgerLayoutTest.java @@ -87,8 +87,8 @@ public class ZkLedgerLayoutTest extends BookKeeperClusterTestCase { ClientConfiguration conf = new ClientConfiguration(); // write bad version ledger layout writeLedgerLayout(conf.getZkLedgersRootPath(), - FlatLedgerManagerFactory.class.getName(), - FlatLedgerManagerFactory.CUR_VERSION, + HierarchicalLedgerManagerFactory.class.getName(), + HierarchicalLedgerManagerFactory.CUR_VERSION, LedgerLayout.LAYOUT_FORMAT_VERSION + 1); ZkLayoutManager zkLayoutManager = new ZkLayoutManager(zkc, conf.getZkLedgersRootPath(), Ids.OPEN_ACL_UNSAFE); @@ -130,7 +130,7 @@ public class ZkLedgerLayoutTest extends BookKeeperClusterTestCase { // write bad format ledger layout StringBuilder sb = new StringBuilder(); sb.append(LedgerLayout.LAYOUT_FORMAT_VERSION).append("\n") - .append(FlatLedgerManagerFactory.class.getName()); + .append(HierarchicalLedgerManagerFactory.class.getName()); zkc.create(ledgersLayout, sb.toString().getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); ZkLayoutManager zkLayoutManager = new ZkLayoutManager(zkc, rootPath, Ids.OPEN_ACL_UNSAFE); @@ -148,15 +148,15 @@ public class ZkLedgerLayoutTest extends BookKeeperClusterTestCase { ClientConfiguration conf = new ClientConfiguration(); // write v1 ledger layout writeLedgerLayout(conf.getZkLedgersRootPath(), - FlatLedgerManagerFactory.NAME, - FlatLedgerManagerFactory.CUR_VERSION, 1); + HierarchicalLedgerManagerFactory.NAME, + HierarchicalLedgerManagerFactory.CUR_VERSION, 1); ZkLayoutManager zkLayoutManager = new ZkLayoutManager(zkc, conf.getZkLedgersRootPath(), Ids.OPEN_ACL_UNSAFE); LedgerLayout layout = zkLayoutManager.readLedgerLayout(); assertNotNull("Should not be null", layout); - assertEquals(FlatLedgerManagerFactory.NAME, layout.getManagerFactoryClass()); - assertEquals(FlatLedgerManagerFactory.CUR_VERSION, layout.getManagerVersion()); + assertEquals(HierarchicalLedgerManagerFactory.NAME, layout.getManagerFactoryClass()); + assertEquals(HierarchicalLedgerManagerFactory.CUR_VERSION, layout.getManagerVersion()); assertEquals(1, layout.getLayoutFormatVersion()); } } diff --git a/deploy/kubernetes/gke/bookkeeper.statefulset.yml b/deploy/kubernetes/gke/bookkeeper.statefulset.yml index d72cb2c..03ea637 100644 --- a/deploy/kubernetes/gke/bookkeeper.statefulset.yml +++ b/deploy/kubernetes/gke/bookkeeper.statefulset.yml @@ -39,8 +39,6 @@ data: BK_ledgerDirectories: "/bookkeeper/data/ledgers" BK_indexDirectories: "/bookkeeper/data/ledgers" BK_zkServers: zookeeper - # the default manager is flat, which is not good for supporting large number of ledgers - BK_ledgerManagerType: "hierarchical" # TODO: Issue 458: https://github.com/apache/bookkeeper/issues/458 #BK_statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider # use hostname as bookie id for StatefulSets deployment diff --git a/deploy/kubernetes/gke/bookkeeper.yaml b/deploy/kubernetes/gke/bookkeeper.yaml index 5876294..7ad58af 100644 --- a/deploy/kubernetes/gke/bookkeeper.yaml +++ b/deploy/kubernetes/gke/bookkeeper.yaml @@ -30,8 +30,6 @@ data: BK_ledgerDirectories: "/bookkeeper/data/ledgers" BK_indexDirectories: "/bookkeeper/data/ledgers" BK_zkServers: zookeeper - # the default manager is flat, which is not good for supporting large number of ledgers - BK_ledgerManagerType: "hierarchical" # TODO: Issue 458: https://github.com/apache/bookkeeper/issues/458 #BK_statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider --- diff --git a/site/_data/config/bk_server.yaml b/site/_data/config/bk_server.yaml index 54a26c4..30882c8 100644 --- a/site/_data/config/bk_server.yaml +++ b/site/_data/config/bk_server.yaml @@ -270,8 +270,8 @@ groups: - name: Ledger manager settings params: - - param: ledgerManagerType - description: The ledger manager type, which defines how ledgers are stored, managed, and garbage collected. See the [Ledger Manager](../../getting-started/concepts#ledger-manager) guide for more details. + - param: ledgerManagerFactoryClass + description: The ledger manager factory class, which defines how ledgers are stored, managed, and garbage collected. See the [Ledger Manager](../../getting-started/concepts#ledger-manager) guide for more details. default: flat - param: zkLedgersRootPath description: Root Zookeeper path to store ledger metadata. This parameter is used by zookeeper-based ledger manager as a root znode to store all ledgers. diff --git a/site/docs/latest/overview/releaseNotes.md b/site/docs/latest/overview/releaseNotes.md index 45f9289..259da6e 100644 --- a/site/docs/latest/overview/releaseNotes.md +++ b/site/docs/latest/overview/releaseNotes.md @@ -10,3 +10,9 @@ If you want to learn the progress of `{{ site.latest_version }}`, you can do: - Or [subscribe](mailto:dev-subscr...@bookkeeper.apache.org) the [d...@bookkeeper.apache.org](mailto:d...@bookkeeper.apache.org) to join development discussions, propose new ideas and connect with contributors. - Or [join us on Slack](https://apachebookkeeper.herokuapp.com/) to connect with Apache BookKeeper committers and contributors. + +### Existing API changes + +- The default ledger manager factory is changed from FlatLedgerManagerFactory to HierarchicalLedgerManagerFactory if `ledgerManagerFactoryClass` + is specified. If you have a cluster running with older versions and no `ledgerManagerFactoryClass` is set before, you need to set + `ledgerManagerFactoryClass` explicitly to `org.apache.bookkeeper.meta.FlatLedgerManagerFactory` in your bookie configuration before upgrade. -- To stop receiving notification emails like this one, please contact si...@apache.org.