This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new b2439d6 Utility to dump/restore ledger metadata to/from file
b2439d6 is described below
commit b2439d67073c880d350f93c1939bbede5e00ec49
Author: Ivan Kelly <[email protected]>
AuthorDate: Fri Sep 14 08:47:26 2018 +0200
Utility to dump/restore ledger metadata to/from file
Dumping will dump the serialized metadata to a file. Restoring will
parse the metadata from the file and write it to the metadata store.
Also changes the LedgerIdFormatter to default to long formatter (it
was UUID before, and we don't even currently support UUID ledger ids).
This fixes #1604
Author: Ivan Kelly <[email protected]>
Author: Enrico Olivelli <[email protected]>
Author: cguttapalem <[email protected]>
Author: Qi Wang <[email protected]>
Reviewers: Enrico Olivelli <[email protected]>, Sijie Guo
<[email protected]>, Jia Zhai <None>
This closes #1667 from ivankelly/dump-metadata-to-from-file
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 55 +++++++++++-----------
.../bookkeeper/conf/AbstractConfiguration.java | 2 +-
.../apache/bookkeeper/util/LedgerIdFormatter.java | 4 +-
.../tests/integration/TestBookieShellCluster.java | 43 +++++++++++++++++
4 files changed, 73 insertions(+), 31 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 8eebce5..a6e0f44 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -25,9 +25,9 @@ import static
org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithRegistra
import static
org.apache.bookkeeper.tools.cli.helpers.CommandHelpers.getBookieSocketAddrStringRepresentation;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.UncheckedExecutionException;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
@@ -105,6 +105,7 @@ import org.apache.bookkeeper.proto.BookieClient;
import org.apache.bookkeeper.proto.BookieClientImpl;
import org.apache.bookkeeper.proto.BookieProtocol;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
+import
org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallbackFuture;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.Processor;
import org.apache.bookkeeper.replication.AuditorElector;
import org.apache.bookkeeper.replication.ReplicationException;
@@ -1084,28 +1085,6 @@ public class BookieShell implements Tool {
}
}
- static class ReadMetadataCallback extends AbstractFuture<LedgerMetadata>
- implements GenericCallback<LedgerMetadata> {
- final long ledgerId;
-
- ReadMetadataCallback(long ledgerId) {
- this.ledgerId = ledgerId;
- }
-
- long getLedgerId() {
- return ledgerId;
- }
-
- @Override
- public void operationComplete(int rc, LedgerMetadata result) {
- if (rc != 0) {
- setException(BKException.create(rc));
- } else {
- set(result);
- }
- }
- }
-
/**
* Print the metadata for a ledger.
*/
@@ -1115,6 +1094,8 @@ public class BookieShell implements Tool {
LedgerMetadataCmd() {
super(CMD_LEDGERMETADATA);
lOpts.addOption("l", "ledgerid", true, "Ledger ID");
+ lOpts.addOption("dumptofile", true, "Dump metadata for ledger, to
a file");
+ lOpts.addOption("restorefromfile", true, "Restore metadata for
ledger, from a file");
}
@Override
@@ -1125,11 +1106,29 @@ public class BookieShell implements Tool {
return -1;
}
+ if (cmdLine.hasOption("dumptofile") &&
cmdLine.hasOption("restorefromfile")) {
+ System.err.println("Only one of --dumptofile and
--restorefromfile can be specified");
+ return -2;
+ }
runFunctionWithLedgerManagerFactory(bkConf, mFactory -> {
try (LedgerManager m = mFactory.newLedgerManager()) {
- ReadMetadataCallback cb = new ReadMetadataCallback(lid);
- m.readLedgerMetadata(lid, cb);
- printLedgerMetadata(lid, cb.get(), true);
+ if (cmdLine.hasOption("dumptofile")) {
+ GenericCallbackFuture<LedgerMetadata> cb = new
GenericCallbackFuture<>();
+ m.readLedgerMetadata(lid, cb);
+
Files.write(FileSystems.getDefault().getPath(cmdLine.getOptionValue("dumptofile")),
+ cb.join().serialize());
+ } else if (cmdLine.hasOption("restorefromfile")) {
+ byte[] serialized = Files.readAllBytes(
+
FileSystems.getDefault().getPath(cmdLine.getOptionValue("restorefromfile")));
+ LedgerMetadata md =
LedgerMetadata.parseConfig(serialized, Version.NEW, Optional.absent());
+ GenericCallbackFuture<LedgerMetadata> cb = new
GenericCallbackFuture<>();
+ m.createLedgerMetadata(lid, md, cb);
+ cb.join();
+ } else {
+ GenericCallbackFuture<LedgerMetadata> cb = new
GenericCallbackFuture<>();
+ m.readLedgerMetadata(lid, cb);
+ printLedgerMetadata(lid, cb.get(), true);
+ }
} catch (Exception e) {
throw new UncheckedExecutionException(e);
}
@@ -1141,12 +1140,12 @@ public class BookieShell implements Tool {
@Override
String getDescription() {
- return "Print the metadata for a ledger.";
+ return "Print the metadata for a ledger, or optionally dump to a
file.";
}
@Override
String getUsage() {
- return "ledgermetadata -ledgerid <ledgerid>";
+ return "ledgermetadata -ledgerid <ledgerid> [--dump-to-file
FILENAME|--restore-from-file FILENAME]";
}
@Override
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
index 9f9547c..b72b0ba 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
@@ -105,7 +105,7 @@ public abstract class AbstractConfiguration<T extends
AbstractConfiguration>
// Default formatter classes
protected static final Class<? extends EntryFormatter>
DEFAULT_ENTRY_FORMATTER = StringEntryFormatter.class;
protected static final Class<? extends LedgerIdFormatter>
DEFAULT_LEDGERID_FORMATTER =
- LedgerIdFormatter.UUIDLedgerIdFormatter.class;
+ LedgerIdFormatter.LongLedgerIdFormatter.class;
/**
* This list will be passed to {@link
SSLEngine#setEnabledCipherSuites(java.lang.String[]) }.
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LedgerIdFormatter.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LedgerIdFormatter.java
index ec52cdf..f037f6d 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LedgerIdFormatter.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LedgerIdFormatter.java
@@ -62,8 +62,8 @@ public abstract class LedgerIdFormatter {
formatter = ReflectionUtils.newInstance(ledgerIdFormatterClass);
} catch (Exception e) {
LOG.warn("No formatter class found", e);
- LOG.warn("Using Default UUID Formatter.");
- formatter = new UUIDLedgerIdFormatter();
+ LOG.warn("Using Default Long Formatter.");
+ formatter = new LongLedgerIdFormatter();
}
return formatter;
}
diff --git
a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java
b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java
index 47fbc55..c68a3e1 100644
---
a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java
+++
b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java
@@ -22,6 +22,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertTrue;
import com.github.dockerjava.api.DockerClient;
+import java.util.concurrent.ExecutionException;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.BKException;
@@ -182,4 +183,46 @@ public class TestBookieShellCluster extends
BookieShellTestBase {
}
}
+ @Test
+ public void test102_DumpRestoreMetadata() throws Exception {
+ String zookeeper = String.format("zk+hierarchical://%s/ledgers",
+
BookKeeperClusterUtils.zookeeperConnectString(docker));
+ int numEntries = 100;
+
+ try (BookKeeper bk = BookKeeper.newBuilder(
+ new
ClientConfiguration().setMetadataServiceUri(zookeeper)).build()) {
+ log.info("Writing entries");
+ long ledgerId = writeNEntries(bk, numEntries, 1);
+
+ log.info("Dumping ledger metadata to file");
+ String bookie = BookKeeperClusterUtils.getAnyBookie();
+ String dumpFile = String.format("/tmp/ledger-%d-%d", ledgerId,
System.nanoTime());
+ DockerUtils.runCommand(docker, bookie,
+ bkScript, "shell", "ledgermetadata",
+ "--ledgerid", String.valueOf(ledgerId),
+ "--dumptofile", dumpFile);
+
+ log.info("Delete the ledger metadata");
+ bk.newDeleteLedgerOp().withLedgerId(ledgerId).execute().get();
+
+ // hopefully ledger gc doesn't kick in
+ log.info("Verify that we cannot open ledger");
+ try {
+ validateNEntries(bk, ledgerId, numEntries);
+ Assert.fail("Shouldn't have been able to find anything");
+ } catch (ExecutionException ee) {
+ Assert.assertEquals(ee.getCause().getClass(),
BKException.BKNoSuchLedgerExistsException.class);
+ }
+
+ log.info("Restore the ledger metadata");
+ DockerUtils.runCommand(docker, bookie,
+ bkScript, "shell", "ledgermetadata",
+ "--ledgerid", String.valueOf(ledgerId),
+ "--restorefromfile", dumpFile);
+
+ log.info("Validate that we can read back, after regeneration");
+ validateNEntries(bk, ledgerId, numEntries);
+ }
+ }
+
}
