This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 8cc7239 Issue #1982: Migrate command `bookiesanity`
8cc7239 is described below
commit 8cc7239ac95ea1673b188d57a89aa39d4ecd3c8f
Author: Yong Zhang <[email protected]>
AuthorDate: Mon Mar 18 16:11:27 2019 +0800
Issue #1982: Migrate command `bookiesanity`
Descriptions of the changes in this PR:
- migrate command `bookiesanity`
### Changes
- use command by `bkctl bookie sanity`
Master Issue: #1982
Reviewers: Jia Zhai <[email protected]>, Sijie Guo <[email protected]>
This closes #1983 from zymap/command-bookiesanity, closes #1982
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 63 +-------
.../cli/commands/bookie/SanityTestCommand.java | 139 +++++++++++++++++
.../tools/cli/commands/BookieCommandGroup.java | 2 +
.../cli/commands/bookie/SanityTestCommandTest.java | 164 +++++++++++++++++++++
4 files changed, 310 insertions(+), 58 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 026c900..3d78c33 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -52,7 +52,6 @@ import java.util.Base64;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.Enumeration;
import java.util.Formatter;
import java.util.HashMap;
import java.util.HashSet;
@@ -83,7 +82,6 @@ import
org.apache.bookkeeper.bookie.storage.ldb.LocationsIndexRebuildOp;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BKException.MetaStoreException;
import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.BookKeeperAdmin;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
@@ -113,6 +111,7 @@ import
org.apache.bookkeeper.tools.cli.commands.bookie.FormatCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.InitCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.LastMarkCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.LedgerCommand;
+import org.apache.bookkeeper.tools.cli.commands.bookie.SanityTestCommand;
import org.apache.bookkeeper.tools.cli.commands.bookies.InfoCommand;
import org.apache.bookkeeper.tools.cli.commands.bookies.ListBookiesCommand;
import org.apache.bookkeeper.tools.cli.commands.client.SimpleTestCommand;
@@ -1269,62 +1268,10 @@ public class BookieShell implements Tool {
@Override
int runCmd(CommandLine cmdLine) throws Exception {
- int numberOfEntries = getOptionIntValue(cmdLine, "entries", 10);
- int timeoutSecs = getOptionIntValue(cmdLine, "timeout", 1);
-
- ClientConfiguration conf = new ClientConfiguration();
- conf.addConfiguration(bkConf);
-
conf.setEnsemblePlacementPolicy(LocalBookieEnsemblePlacementPolicy.class);
- conf.setAddEntryTimeout(timeoutSecs);
- conf.setReadEntryTimeout(timeoutSecs);
-
- BookKeeper bk = new BookKeeper(conf);
- LedgerHandle lh = null;
- try {
- lh = bk.createLedger(1, 1, DigestType.MAC, new byte[0]);
- LOG.info("Created ledger {}", lh.getId());
-
- for (int i = 0; i < numberOfEntries; i++) {
- String content = "entry-" + i;
- lh.addEntry(content.getBytes(UTF_8));
- }
-
- LOG.info("Written {} entries in ledger {}", numberOfEntries,
lh.getId());
-
- // Reopen the ledger and read entries
- lh = bk.openLedger(lh.getId(), DigestType.MAC, new byte[0]);
- if (lh.getLastAddConfirmed() != (numberOfEntries - 1)) {
- throw new Exception("Invalid last entry found on ledger.
expecting: " + (numberOfEntries - 1)
- + " -- found: " + lh.getLastAddConfirmed());
- }
-
- Enumeration<LedgerEntry> entries = lh.readEntries(0,
numberOfEntries - 1);
- int i = 0;
- while (entries.hasMoreElements()) {
- LedgerEntry entry = entries.nextElement();
- String actualMsg = new String(entry.getEntry(), UTF_8);
- String expectedMsg = "entry-" + (i++);
- if (!expectedMsg.equals(actualMsg)) {
- throw new Exception("Failed validation of received
message - Expected: " + expectedMsg
- + ", Actual: " + actualMsg);
- }
- }
-
- LOG.info("Read {} entries from ledger {}", entries,
lh.getId());
- } catch (Exception e) {
- LOG.warn("Error in bookie sanity test", e);
- return -1;
- } finally {
- if (lh != null) {
- bk.deleteLedger(lh.getId());
- LOG.info("Deleted ledger {}", lh.getId());
- }
-
- bk.close();
- }
-
- LOG.info("Bookie sanity test succeeded");
- return 0;
+ SanityTestCommand command = new SanityTestCommand();
+ SanityTestCommand.SanityFlags flags = new
SanityTestCommand.SanityFlags();
+ boolean result = command.apply(bkConf, flags);
+ return (result) ? 0 : -1;
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/SanityTestCommand.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/SanityTestCommand.java
new file mode 100644
index 0000000..c1a81fd
--- /dev/null
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/SanityTestCommand.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookie;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.Parameter;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import java.util.Enumeration;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.apache.bookkeeper.bookie.LocalBookieEnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import
org.apache.bookkeeper.tools.cli.commands.bookie.SanityTestCommand.SanityFlags;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommand;
+import org.apache.bookkeeper.tools.framework.CliFlags;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A bookie command to sanity test for local bookie.
+ */
+public class SanityTestCommand extends BookieCommand<SanityFlags> {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(SanityTestCommand.class);
+ private static final String NAME = "sanitytest";
+ private static final String DESC = "Sanity test for local bookie. "
+ + "Create ledger and write/reads
entries on local bookie.";
+
+ public SanityTestCommand() {
+ this(new SanityFlags());
+ }
+
+ public SanityTestCommand(SanityFlags flags) {
+
super(CliSpec.<SanityFlags>newBuilder().withFlags(flags).withName(NAME).withDescription(DESC).build());
+ }
+
+ /**
+ * Flags for sanity command.
+ */
+ @Accessors(fluent = true)
+ @Setter
+ public static class SanityFlags extends CliFlags{
+
+ @Parameter(names = {"-e", "--entries"}, description = "Total entries
to be added for the test (default 10)")
+ private int entries = 10;
+
+ @Parameter(names = { "-t",
+ "--timeout" }, description = "Timeout for write/read operations in
seconds (default 1)")
+ private int timeout = 1;
+
+ }
+
+ @Override
+ public boolean apply(ServerConfiguration conf, SanityFlags cmdFlags) {
+ try {
+ return handle(conf, cmdFlags);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ }
+
+ private boolean handle(ServerConfiguration conf, SanityFlags cmdFlags)
throws Exception {
+ ClientConfiguration clientConf = new ClientConfiguration();
+ clientConf.addConfiguration(conf);
+
clientConf.setEnsemblePlacementPolicy(LocalBookieEnsemblePlacementPolicy.class);
+ clientConf.setAddEntryTimeout(cmdFlags.timeout);
+ clientConf.setReadEntryTimeout(cmdFlags.timeout);
+
+ BookKeeper bk = new BookKeeper(clientConf);
+ LedgerHandle lh = null;
+ try {
+ lh = bk.createLedger(1, 1, BookKeeper.DigestType.MAC, new byte[0]);
+ LOG.info("Create ledger {}", lh.getId());
+
+ for (int i = 0; i < cmdFlags.entries; i++) {
+ String content = "entry-" + i;
+ lh.addEntry(content.getBytes(UTF_8));
+ }
+
+ LOG.info("Written {} entries in ledger {}", cmdFlags.entries,
lh.getId());
+
+ // Reopen the ledger and read entries
+ lh = bk.openLedger(lh.getId(), BookKeeper.DigestType.MAC, new
byte[0]);
+ if (lh.getLastAddConfirmed() != (cmdFlags.entries - 1)) {
+ throw new Exception("Invalid last entry found on ledger.
expecting: " + (cmdFlags.entries - 1)
+ + " -- found: " +
lh.getLastAddConfirmed());
+ }
+
+ Enumeration<LedgerEntry> entries = lh.readEntries(0,
cmdFlags.entries - 1);
+ int i = 0;
+ while (entries.hasMoreElements()) {
+ LedgerEntry entry = entries.nextElement();
+ String actualMsg = new String(entry.getEntry(), UTF_8);
+ String expectedMsg = "entry-" + (i++);
+ if (!expectedMsg.equals(actualMsg)) {
+ throw new Exception("Failed validation of received message
- Expected: " + expectedMsg
+ + ", Actual: " + actualMsg);
+ }
+ }
+
+ LOG.info("Read {} entries from ledger {}", i, lh.getId());
+ } catch (Exception e) {
+ LOG.warn("Error in bookie sanity test", e);
+ return false;
+ } finally {
+ if (lh != null) {
+ bk.deleteLedger(lh.getId());
+ LOG.info("Deleted ledger {}", lh.getId());
+ }
+
+ bk.close();
+ }
+
+ LOG.info("Bookie sanity test succeeded");
+ return true;
+ }
+}
diff --git
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
index e7daee6..940d482 100644
---
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
+++
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
@@ -25,6 +25,7 @@ import
org.apache.bookkeeper.tools.cli.commands.bookie.FormatCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.InitCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.LastMarkCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.LedgerCommand;
+import org.apache.bookkeeper.tools.cli.commands.bookie.SanityTestCommand;
import org.apache.bookkeeper.tools.common.BKFlags;
import org.apache.bookkeeper.tools.framework.CliCommandGroup;
import org.apache.bookkeeper.tools.framework.CliSpec;
@@ -45,6 +46,7 @@ public class BookieCommandGroup extends
CliCommandGroup<BKFlags> {
.addCommand(new LastMarkCommand())
.addCommand(new InitCommand())
.addCommand(new FormatCommand())
+ .addCommand(new SanityTestCommand())
.addCommand(new LedgerCommand())
.build();
diff --git
a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/SanityTestCommandTest.java
b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/SanityTestCommandTest.java
new file mode 100644
index 0000000..fb596ef
--- /dev/null
+++
b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/SanityTestCommandTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookie;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static junit.framework.TestCase.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.verifyNew;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import java.util.Enumeration;
+import java.util.Vector;
+import org.apache.bookkeeper.bookie.LocalBookieEnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommandTestBase;
+import org.apache.commons.configuration.Configuration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+/**
+ * Test for sanity command.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ SanityTestCommand.class, LedgerEntry.class})
+public class SanityTestCommandTest extends BookieCommandTestBase {
+
+ private ClientConfiguration clientConf;
+ private BookKeeper bk;
+ private LedgerHandle lh;
+
+ public SanityTestCommandTest() {
+ super(3, 1);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ clientConf = mock(ClientConfiguration.class);
+
PowerMockito.whenNew(ClientConfiguration.class).withNoArguments().thenReturn(clientConf);
+
+ bk = mock(BookKeeper.class);
+ lh = mock(LedgerHandle.class);
+
PowerMockito.whenNew(BookKeeper.class).withParameterTypes(ClientConfiguration.class)
+ .withArguments(any(ClientConfiguration.class)).thenReturn(bk);
+ when(bk.createLedger(anyInt(), anyInt(),
any(BookKeeper.DigestType.class), eq(new byte[0]))).thenReturn(lh);
+ when(bk.openLedger(anyLong(), any(BookKeeper.DigestType.class), eq(new
byte[0]))).thenReturn(lh);
+ when(lh.getLastAddConfirmed()).thenReturn(9L);
+ Enumeration<LedgerEntry> entryEnumeration = getEntry();
+ when(lh.readEntries(anyLong(),
anyLong())).thenReturn(entryEnumeration);
+ when(lh.getId()).thenReturn(1L);
+
+ }
+
+ private Enumeration<LedgerEntry> getEntry() {
+ Vector<LedgerEntry> entries = new Vector<>();
+ for (int i = 0; i < 10; i++) {
+ LedgerEntry ledgerEntry = mock(LedgerEntry.class);
+ String payload = "entry-" + i;
+ when(ledgerEntry.getEntry()).thenReturn(payload.getBytes(UTF_8));
+ entries.add(ledgerEntry);
+ }
+ return entries.elements();
+ }
+
+ @Test
+ public void testDefaultArgs() {
+ testSanityCommand("");
+ }
+
+ @Test
+ public void testEntriesShortArgs() {
+ when(lh.getLastAddConfirmed()).thenReturn(0L);
+ testSanityCommand("-e", "1");
+ verifyFunc();
+ }
+
+ @Test
+ public void testEntriesLongArgs() {
+ when(lh.getLastAddConfirmed()).thenReturn(0L);
+ testSanityCommand("--entries", "1");
+ verifyFunc();
+ }
+
+ private void verifyFunc() {
+ try {
+ verify(clientConf, times(1)).setAddEntryTimeout(1);
+ verify(clientConf, times(1)).setReadEntryTimeout(1);
+ verify(lh, times(1)).addEntry(any());
+ verify(lh, times(1)).readEntries(0, 0);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ }
+
+ @Test
+ public void testTimeoutShortArgs() {
+ testSanityCommand("-t", "10");
+ }
+
+ @Test
+ public void testTimeoutLongArgs() {
+ testSanityCommand("--timeout", "10");
+ }
+
+ private void verifyTimeout() {
+ verify(clientConf, times(1)).setAddEntryTimeout(10);
+ verify(clientConf, times(1)).setReadEntryTimeout(10);
+ try {
+ verify(lh, times(10)).addEntry(any());
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ }
+
+ public void testSanityCommand(String... args) {
+ SanityTestCommand cmd = new SanityTestCommand();
+ assertTrue(cmd.apply(bkFlags, args));
+ try {
+ verifyNew(ClientConfiguration.class, times(1)).withNoArguments();
+ verify(clientConf,
times(1)).addConfiguration(any(Configuration.class));
+ verify(clientConf,
times(1)).setEnsemblePlacementPolicy(LocalBookieEnsemblePlacementPolicy.class);
+
+ verifyNew(BookKeeper.class).withArguments(clientConf);
+ verify(bk, times(1)).createLedger(1, 1, BookKeeper.DigestType.MAC,
new byte[0]);
+ verify(lh, times(6)).getId();
+ verify(bk, times(1)).openLedger(anyLong(),
eq(BookKeeper.DigestType.MAC), eq(new byte[0]));
+ verify(lh, times(1)).getLastAddConfirmed();
+ verify(bk, times(1)).deleteLedger(anyLong());
+ verify(bk, times(1)).close();
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ }
+}
