This is an automated email from the ASF dual-hosted git repository. ivank pushed a commit to branch bk-replace-bookieid in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 0af5f5a5f517fcc07af55c447fd36590dca4fbbe Author: Ivan Kelly <[email protected]> AuthorDate: Fri Mar 1 11:58:02 2019 +0100 A tools to search and replace bookie ids in ledger metadata To use: ``` bin/bkctl bookieid searchreplace --from <from> --to <to> ``` --- .../bookkeeper/tests/integration/TestCLI.java | 49 ++++++++ .../bookkeeper/tests/integration/TestSmoke.java | 8 +- .../tools/cli/commands/BookieIdCommandGroup.java | 48 ++++++++ .../bookieid/SearchReplaceBookieIdCommand.java | 128 +++++++++++++++++++++ .../tools/cli/commands/bookieid/package-info.java | 22 ++++ tools/ledger/src/main/resources/commands | 1 + 6 files changed, 252 insertions(+), 4 deletions(-) diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java index f31beef..781d28e 100644 --- a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java @@ -19,9 +19,15 @@ package org.apache.bookkeeper.tests.integration; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import com.github.dockerjava.api.DockerClient; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.client.api.WriteHandle; +import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils; import org.apache.bookkeeper.tests.integration.utils.DockerUtils; import org.jboss.arquillian.junit.Arquillian; @@ -102,4 +108,47 @@ public class TestCLI { ).contains("ReadWrite Bookies :")); } + @Test + public void test004_SearchReplaceBookieId() throws Exception { + String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker); + + String bookie = BookKeeperClusterUtils.getAnyBookie(); + int numEntries = 100; + try (BookKeeper bk = new BookKeeper(zookeeper)) { + long ledgerId; + BookieSocketAddress toReplace; + BookieSocketAddress replaceWith = new BookieSocketAddress("192.0.2.1:3181"); + try (WriteHandle writelh = bk.newCreateLedgerOp() + .withDigestType(DigestType.CRC32C).withPassword(TestSmoke.PASSWD) + .withEnsembleSize(1).withWriteQuorumSize(1).withAckQuorumSize(1).execute().get()) { + ledgerId = writelh.getId(); + toReplace = writelh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + for (int i = 0; i < numEntries; i++) { + writelh.append(("entry-" + i).getBytes()); + } + } + + TestSmoke.readEntries(bk, ledgerId, numEntries); + + DockerUtils.runCommand(docker, bookie, + bkctl, + "bookieid", "searchreplace", + "--from", toReplace.toString(), + "--to", replaceWith.toString()); + + try { + TestSmoke.readEntries(bk, ledgerId, numEntries); + fail("Shouldn't be able to read, as bookie id is rubbish"); + } catch (BKException.BKBookieHandleNotAvailableException e) { + // expected + } + + DockerUtils.runCommand(docker, bookie, + bkctl, + "bookieid", "searchreplace", + "--from", replaceWith.toString(), + "--to", toReplace.toString()); + TestSmoke.readEntries(bk, ledgerId, numEntries); + } + } } diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java index 2662b18..1efa63d 100644 --- a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java @@ -57,7 +57,7 @@ import org.junit.runners.MethodSorters; @RunWith(Arquillian.class) @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class TestSmoke { - private static final byte[] PASSWD = "foobar".getBytes(); + static final byte[] PASSWD = "foobar".getBytes(); @ArquillianResource DockerClient docker; @@ -112,9 +112,9 @@ public class TestSmoke { } } - private static void readEntries(BookKeeper bk, - long ledgerId, - int numExpectedEntries) throws Exception { + static void readEntries(BookKeeper bk, + long ledgerId, + int numExpectedEntries) throws Exception { try (LedgerHandle readlh = bk.openLedger(ledgerId, BookKeeper.DigestType.CRC32C, PASSWD)) { long lac = readlh.getLastAddConfirmed(); int i = 0; diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java new file mode 100644 index 0000000..f230d09 --- /dev/null +++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.tools.cli.commands; + +import static org.apache.bookkeeper.tools.common.BKCommandCategories.CATEGORY_INFRA_SERVICE; + +import org.apache.bookkeeper.tools.cli.BKCtl; +import org.apache.bookkeeper.tools.cli.commands.bookieid.SearchReplaceBookieIdCommand; +import org.apache.bookkeeper.tools.common.BKFlags; +import org.apache.bookkeeper.tools.framework.CliCommandGroup; +import org.apache.bookkeeper.tools.framework.CliSpec; + +/** + * Commands that operate on bookie IDs. + */ +public class BookieIdCommandGroup extends CliCommandGroup<BKFlags> { + + private static final String NAME = "bookieid"; + private static final String DESC = "Commands operating on bookie ids"; + + private static final CliSpec<BKFlags> spec = CliSpec.<BKFlags>newBuilder() + .withName(NAME) + .withDescription(DESC) + .withParent(BKCtl.NAME) + .withCategory(CATEGORY_INFRA_SERVICE) + .addCommand(new SearchReplaceBookieIdCommand()) + .build(); + + public BookieIdCommandGroup() { + super(spec); + } +} diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java new file mode 100644 index 0000000..2e4f1e7 --- /dev/null +++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.tools.cli.commands.bookieid; + +import com.beust.jcommander.Parameter; +import com.google.common.util.concurrent.RateLimiter; + +import java.util.ArrayList; +import java.util.List; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.api.BookKeeper; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.tools.cli.helpers.ClientCommand; +import org.apache.bookkeeper.tools.framework.CliFlags; +import org.apache.bookkeeper.tools.framework.CliSpec; +import org.apache.bookkeeper.versioning.Versioned; + +/** + * Search and replace a bookie id in ledger metadata. + */ +public class SearchReplaceBookieIdCommand extends ClientCommand<SearchReplaceBookieIdCommand.Flags> { + + private static final String NAME = "searchreplace"; + private static final String DESC = "Search all ledgers for a bookie ID and replace"; + + /** + * Flags for replace bookie id. + */ + @Accessors(fluent = true) + @Setter + public static class Flags extends CliFlags { + + @Parameter(names = { "-f", "--from" }, description = "Bookie ID to search for", required = true) + private String from; + @Parameter(names = { "-t", "--to" }, description = "Bookie ID to replace with", required = true) + private String to; + @Parameter(names = { "-m", "--max" }, description = "Maximum number of replacements to make") + private long max = Long.MAX_VALUE; + @Parameter(names = { "-r", "--rate" }, description = "Rate limit (updates per second)") + private int rate = Integer.MAX_VALUE; + @Parameter(names = { "--dry-run" }, description = "Don't actually write anything") + private boolean dryRun = false; + @Parameter(names = { "-v", "--verbose" }, description = "Verbose output") + private boolean verbose = false; + } + + public SearchReplaceBookieIdCommand() { + this(new Flags()); + } + + public SearchReplaceBookieIdCommand(Flags flags) { + super(CliSpec.<Flags>newBuilder() + .withName(NAME) + .withDescription(DESC) + .withFlags(flags) + .build()); + } + + @Override + protected void run(BookKeeper bk, Flags flags) throws Exception { + BookKeeperAdmin admin = new BookKeeperAdmin((org.apache.bookkeeper.client.BookKeeper) bk); + LedgerManager ledgerManager = ((org.apache.bookkeeper.client.BookKeeper) bk).getLedgerManager(); + long i = 0; + + BookieSocketAddress fromAddr = new BookieSocketAddress(flags.from); + BookieSocketAddress toAddr = new BookieSocketAddress(flags.to); + System.out.println(String.format("Replacing bookie id %s with %s in metadata", fromAddr, toAddr)); + RateLimiter limiter = RateLimiter.create(flags.rate); + for (Long lid : admin.listLedgers()) { + Versioned<LedgerMetadata> md = ledgerManager.readLedgerMetadata(lid).get(); + if (md.getValue().getAllEnsembles().entrySet().stream().anyMatch(e -> e.getValue().contains(fromAddr))) { + limiter.acquire(); + + LedgerMetadataBuilder builder = LedgerMetadataBuilder.from(md.getValue()); + md.getValue().getAllEnsembles().entrySet().stream() + .filter(e -> e.getValue().contains(fromAddr)) + .forEach(e -> { + List<BookieSocketAddress> ensemble = new ArrayList<>(e.getValue()); + ensemble.replaceAll((a) -> { + if (a.equals(fromAddr)) { + return toAddr; + } else { + return a; + } + }); + builder.replaceEnsembleEntry(e.getKey(), ensemble); + }); + LedgerMetadata newMeta = builder.build(); + if (flags.verbose) { + System.out.println("Replacing ledger " + lid + " metadata ..."); + System.out.println(md.getValue().toString()); + System.out.println("with ..."); + System.out.println(newMeta.toString()); + } + i++; + if (!flags.dryRun) { + ledgerManager.writeLedgerMetadata(lid, newMeta, md.getVersion()).get(); + } + } + if (i >= flags.max) { + System.out.println("Max number of ledgers processed, exiting"); + break; + } + } + System.out.println("Replaced bookie ID in " + i + " ledgers"); + } +} diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java new file mode 100644 index 0000000..131d672 --- /dev/null +++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * CLI commands for working with Bookie IDs. + */ +package org.apache.bookkeeper.tools.cli.commands.bookieid; diff --git a/tools/ledger/src/main/resources/commands b/tools/ledger/src/main/resources/commands index 7ea146b..c9af7a1 100644 --- a/tools/ledger/src/main/resources/commands +++ b/tools/ledger/src/main/resources/commands @@ -17,6 +17,7 @@ # org.apache.bookkeeper.tools.cli.commands.BookieCommandGroup +org.apache.bookkeeper.tools.cli.commands.BookieIdCommandGroup org.apache.bookkeeper.tools.cli.commands.BookiesCommandGroup org.apache.bookkeeper.tools.cli.commands.CookieCommandGroup org.apache.bookkeeper.tools.cli.commands.LedgerCommandGroup
