This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new f4cbf8a Tool to search and replace bookie ids in ledger metadata
f4cbf8a is described below
commit f4cbf8a395150e4ea360ef2c8b1cfa95d6d0625f
Author: Ivan Kelly <[email protected]>
AuthorDate: Thu Mar 14 06:19:23 2019 +0100
Tool to search and replace bookie ids in ledger metadata
To use:
```
bin/bkctl bookieid searchreplace --from <from> --to <to>
```
To be used in cases where the DNS name of the bookie has to change, and you
don't want all the data to have to be moved by autorecovery.
Reviewers: Enrico Olivelli <[email protected]>, Jia Zhai
<[email protected]>
This closes #1968 from ivankelly/bk-replace-bookieid
---
.../bookkeeper/tests/integration/TestCLI.java | 49 ++++++++
.../bookkeeper/tests/integration/TestSmoke.java | 8 +-
.../tools/cli/commands/BookieIdCommandGroup.java | 48 ++++++++
.../bookieid/SearchReplaceBookieIdCommand.java | 130 +++++++++++++++++++++
.../tools/cli/commands/bookieid/package-info.java | 22 ++++
tools/ledger/src/main/resources/commands | 1 +
6 files changed, 254 insertions(+), 4 deletions(-)
diff --git
a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
index f31beef..781d28e 100644
---
a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
+++
b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
@@ -19,9 +19,15 @@
package org.apache.bookkeeper.tests.integration;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import com.github.dockerjava.api.DockerClient;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.client.api.WriteHandle;
+import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils;
import org.apache.bookkeeper.tests.integration.utils.DockerUtils;
import org.jboss.arquillian.junit.Arquillian;
@@ -102,4 +108,47 @@ public class TestCLI {
).contains("ReadWrite Bookies :"));
}
+ @Test
+ public void test004_SearchReplaceBookieId() throws Exception {
+ String zookeeper =
BookKeeperClusterUtils.zookeeperConnectString(docker);
+
+ String bookie = BookKeeperClusterUtils.getAnyBookie();
+ int numEntries = 100;
+ try (BookKeeper bk = new BookKeeper(zookeeper)) {
+ long ledgerId;
+ BookieSocketAddress toReplace;
+ BookieSocketAddress replaceWith = new
BookieSocketAddress("192.0.2.1:3181");
+ try (WriteHandle writelh = bk.newCreateLedgerOp()
+
.withDigestType(DigestType.CRC32C).withPassword(TestSmoke.PASSWD)
+
.withEnsembleSize(1).withWriteQuorumSize(1).withAckQuorumSize(1).execute().get())
{
+ ledgerId = writelh.getId();
+ toReplace =
writelh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+ for (int i = 0; i < numEntries; i++) {
+ writelh.append(("entry-" + i).getBytes());
+ }
+ }
+
+ TestSmoke.readEntries(bk, ledgerId, numEntries);
+
+ DockerUtils.runCommand(docker, bookie,
+ bkctl,
+ "bookieid", "searchreplace",
+ "--from", toReplace.toString(),
+ "--to", replaceWith.toString());
+
+ try {
+ TestSmoke.readEntries(bk, ledgerId, numEntries);
+ fail("Shouldn't be able to read, as bookie id is rubbish");
+ } catch (BKException.BKBookieHandleNotAvailableException e) {
+ // expected
+ }
+
+ DockerUtils.runCommand(docker, bookie,
+ bkctl,
+ "bookieid", "searchreplace",
+ "--from", replaceWith.toString(),
+ "--to", toReplace.toString());
+ TestSmoke.readEntries(bk, ledgerId, numEntries);
+ }
+ }
}
diff --git
a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
index 2662b18..1efa63d 100644
---
a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
+++
b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
@@ -57,7 +57,7 @@ import org.junit.runners.MethodSorters;
@RunWith(Arquillian.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class TestSmoke {
- private static final byte[] PASSWD = "foobar".getBytes();
+ static final byte[] PASSWD = "foobar".getBytes();
@ArquillianResource
DockerClient docker;
@@ -112,9 +112,9 @@ public class TestSmoke {
}
}
- private static void readEntries(BookKeeper bk,
- long ledgerId,
- int numExpectedEntries) throws Exception {
+ static void readEntries(BookKeeper bk,
+ long ledgerId,
+ int numExpectedEntries) throws Exception {
try (LedgerHandle readlh = bk.openLedger(ledgerId,
BookKeeper.DigestType.CRC32C, PASSWD)) {
long lac = readlh.getLastAddConfirmed();
int i = 0;
diff --git
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java
new file mode 100644
index 0000000..f230d09
--- /dev/null
+++
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands;
+
+import static
org.apache.bookkeeper.tools.common.BKCommandCategories.CATEGORY_INFRA_SERVICE;
+
+import org.apache.bookkeeper.tools.cli.BKCtl;
+import
org.apache.bookkeeper.tools.cli.commands.bookieid.SearchReplaceBookieIdCommand;
+import org.apache.bookkeeper.tools.common.BKFlags;
+import org.apache.bookkeeper.tools.framework.CliCommandGroup;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+
+/**
+ * Commands that operate on bookie IDs.
+ */
+public class BookieIdCommandGroup extends CliCommandGroup<BKFlags> {
+
+ private static final String NAME = "bookieid";
+ private static final String DESC = "Commands operating on bookie ids";
+
+ private static final CliSpec<BKFlags> spec = CliSpec.<BKFlags>newBuilder()
+ .withName(NAME)
+ .withDescription(DESC)
+ .withParent(BKCtl.NAME)
+ .withCategory(CATEGORY_INFRA_SERVICE)
+ .addCommand(new SearchReplaceBookieIdCommand())
+ .build();
+
+ public BookieIdCommandGroup() {
+ super(spec);
+ }
+}
diff --git
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java
new file mode 100644
index 0000000..d2486b4
--- /dev/null
+++
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookieid;
+
+import com.beust.jcommander.Parameter;
+import com.google.common.util.concurrent.RateLimiter;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
+import org.apache.bookkeeper.client.api.BookKeeper;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.tools.cli.helpers.ClientCommand;
+import org.apache.bookkeeper.tools.framework.CliFlags;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+import org.apache.bookkeeper.versioning.Versioned;
+
+/**
+ * Search and replace a bookie id in ledger metadata.
+ */
+public class SearchReplaceBookieIdCommand extends
ClientCommand<SearchReplaceBookieIdCommand.Flags> {
+
+ private static final String NAME = "searchreplace";
+ private static final String DESC = "Search all ledgers for a bookie ID and
replace";
+
+ /**
+ * Flags for replace bookie id.
+ */
+ @Accessors(fluent = true)
+ @Setter
+ public static class Flags extends CliFlags {
+
+ @Parameter(names = { "-f", "--from" }, description = "Bookie ID to
search for", required = true)
+ private String from;
+ @Parameter(names = { "-t", "--to" }, description = "Bookie ID to
replace with", required = true)
+ private String to;
+ @Parameter(names = { "-m", "--max" }, description = "Maximum number of
replacements to make")
+ private long max = Long.MAX_VALUE;
+ @Parameter(names = { "-r", "--rate" }, description = "Rate limit
(updates per second)")
+ private int rate = Integer.MAX_VALUE;
+ @Parameter(names = { "--dry-run" }, description = "Don't actually
write anything")
+ private boolean dryRun = false;
+ @Parameter(names = { "-v", "--verbose" }, description = "Verbose
output")
+ private boolean verbose = false;
+ }
+
+ public SearchReplaceBookieIdCommand() {
+ this(new Flags());
+ }
+
+ public SearchReplaceBookieIdCommand(Flags flags) {
+ super(CliSpec.<Flags>newBuilder()
+ .withName(NAME)
+ .withDescription(DESC)
+ .withFlags(flags)
+ .build());
+ }
+
+ @Override
+ protected void run(BookKeeper bk, Flags flags) throws Exception {
+ try (BookKeeperAdmin admin = new
BookKeeperAdmin((org.apache.bookkeeper.client.BookKeeper) bk)) {
+ LedgerManager ledgerManager =
((org.apache.bookkeeper.client.BookKeeper) bk).getLedgerManager();
+ long i = 0;
+
+ BookieSocketAddress fromAddr = new BookieSocketAddress(flags.from);
+ BookieSocketAddress toAddr = new BookieSocketAddress(flags.to);
+ System.out.println(String.format("Replacing bookie id %s with %s
in metadata", fromAddr, toAddr));
+ RateLimiter limiter = RateLimiter.create(flags.rate);
+ for (Long lid : admin.listLedgers()) {
+ Versioned<LedgerMetadata> md =
ledgerManager.readLedgerMetadata(lid).get();
+ if (md.getValue().getAllEnsembles().entrySet()
+ .stream().anyMatch(e ->
e.getValue().contains(fromAddr))) {
+ limiter.acquire();
+
+ LedgerMetadataBuilder builder =
LedgerMetadataBuilder.from(md.getValue());
+ md.getValue().getAllEnsembles().entrySet().stream()
+ .filter(e -> e.getValue().contains(fromAddr))
+ .forEach(e -> {
+ List<BookieSocketAddress> ensemble = new
ArrayList<>(e.getValue());
+ ensemble.replaceAll((a) -> {
+ if (a.equals(fromAddr)) {
+ return toAddr;
+ } else {
+ return a;
+ }
+ });
+ builder.replaceEnsembleEntry(e.getKey(),
ensemble);
+ });
+ LedgerMetadata newMeta = builder.build();
+ if (flags.verbose) {
+ System.out.println("Replacing ledger " + lid + "
metadata ...");
+ System.out.println(md.getValue().toSafeString());
+ System.out.println("with ...");
+ System.out.println(newMeta.toSafeString());
+ }
+ i++;
+ if (!flags.dryRun) {
+ ledgerManager.writeLedgerMetadata(lid, newMeta,
md.getVersion()).get();
+ }
+ }
+ if (i >= flags.max) {
+ System.out.println("Max number of ledgers processed,
exiting");
+ break;
+ }
+ }
+ System.out.println("Replaced bookie ID in " + i + " ledgers");
+ }
+ }
+}
diff --git
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java
new file mode 100644
index 0000000..131d672
--- /dev/null
+++
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * CLI commands for working with Bookie IDs.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookieid;
diff --git a/tools/ledger/src/main/resources/commands
b/tools/ledger/src/main/resources/commands
index 7ea146b..c9af7a1 100644
--- a/tools/ledger/src/main/resources/commands
+++ b/tools/ledger/src/main/resources/commands
@@ -17,6 +17,7 @@
#
org.apache.bookkeeper.tools.cli.commands.BookieCommandGroup
+org.apache.bookkeeper.tools.cli.commands.BookieIdCommandGroup
org.apache.bookkeeper.tools.cli.commands.BookiesCommandGroup
org.apache.bookkeeper.tools.cli.commands.CookieCommandGroup
org.apache.bookkeeper.tools.cli.commands.LedgerCommandGroup
