This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 7ad5849 Migrate command `regenerate-interleaved-storage-index-file`
7ad5849 is described below
commit 7ad5849b1b0d9dbf2274a6fd00ad26f7286f7cff
Author: Yong Zhang <[email protected]>
AuthorDate: Tue Apr 9 20:00:26 2019 +0800
Migrate command `regenerate-interleaved-storage-index-file`
Descriptions of the changes in this PR:
#2033
```
Regenerate an interleaved storage index file, from available entrylogger
files.
Usage: bkctl bookie regenerate-interleaved-storage-index-file [flags]
Flags:
-b, --b64password
The password in base64 encoding, for cases where the password is not
UTF-8.
-d, --dryrun
Process the entryLogger, but don't write anthing.
-l, --ledgerids
Ledger(s) whose index needs to be regenerated. Multiple can be
specified, comma separated.
-p, --password
The bookie stores the password in the index file, so we need it to
regenerate.This must match the value in the ledger metadata.
-h, --help
Display help information
```
Reviewers: Sijie Guo <[email protected]>
This closes #2035 from zymap/command-RISIF
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 29 ++---
...generateInterleavedStorageIndexFileCommand.java | 133 +++++++++++++++++++++
.../tools/cli/commands/BookieCommandGroup.java | 2 +
...rateInterleavedStorageIndexFileCommandTest.java | 85 +++++++++++++
4 files changed, 232 insertions(+), 17 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 4446903..4de4aa7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -18,7 +18,6 @@
package org.apache.bookkeeper.bookie;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory;
import static
org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithMetadataBookieDriver;
import static
org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithRegistrationManager;
@@ -39,7 +38,6 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Base64;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -48,7 +46,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.bookkeeper.bookie.BookieException.CookieNotFoundException;
@@ -84,6 +81,7 @@ import
org.apache.bookkeeper.tools.cli.commands.bookie.ReadJournalCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLedgerCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogMetadataCommand;
+import
org.apache.bookkeeper.tools.cli.commands.bookie.RegenerateInterleavedStorageIndexFileCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.SanityTestCommand;
import org.apache.bookkeeper.tools.cli.commands.bookies.DecommissionCommand;
import org.apache.bookkeeper.tools.cli.commands.bookies.InfoCommand;
@@ -2173,23 +2171,20 @@ public class BookieShell implements Tool {
@Override
int runCmd(CommandLine cmdLine) throws Exception {
- byte[] password;
+ RegenerateInterleavedStorageIndexFileCommand cmd = new
RegenerateInterleavedStorageIndexFileCommand();
+ RegenerateInterleavedStorageIndexFileCommand.RISIFFlags
+ flags = new
RegenerateInterleavedStorageIndexFileCommand.RISIFFlags();
+ List<Long> ledgerIds =
Arrays.stream(cmdLine.getOptionValues("ledgerIds")).map((id) ->
Long.parseLong(id))
+ .collect(Collectors.toList());
+ boolean dryRun = cmdLine.hasOption("dryRun");
+ flags.ledgerIds(ledgerIds);
if (cmdLine.hasOption("password")) {
- password = cmdLine.getOptionValue("password").getBytes(UTF_8);
+ flags.password(cmdLine.getOptionValue("password"));
} else if (cmdLine.hasOption("b64password")) {
- password =
Base64.getDecoder().decode(cmdLine.getOptionValue("b64password"));
- } else {
- LOG.error("The password must be specified to regenerate the
index file.");
- return 1;
+ flags.b64Password(cmdLine.getOptionValue("b64password"));
}
- Set<Long> ledgerIds =
Arrays.stream(cmdLine.getOptionValues("ledgerIds"))
- .map((id) -> Long.parseLong(id)).collect(Collectors.toSet());
- boolean dryRun = cmdLine.hasOption("dryRun");
-
- LOG.info("=== Rebuilding index file for {} ===", ledgerIds);
- ServerConfiguration conf = new ServerConfiguration(bkConf);
- new InterleavedStorageRegenerateIndexOp(conf, ledgerIds,
password).initiate(dryRun);
- LOG.info("-- Done rebuilding index file for {} --", ledgerIds);
+ flags.dryRun(dryRun);
+ cmd.apply(bkConf, flags);
return 0;
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/RegenerateInterleavedStorageIndexFileCommand.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/RegenerateInterleavedStorageIndexFileCommand.java
new file mode 100644
index 0000000..bbf933f
--- /dev/null
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/RegenerateInterleavedStorageIndexFileCommand.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookie;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.converters.CommaParameterSplitter;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.apache.bookkeeper.bookie.InterleavedStorageRegenerateIndexOp;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommand;
+import org.apache.bookkeeper.tools.framework.CliFlags;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Command to regenerate an index file for interleaved storage.
+ */
+public class RegenerateInterleavedStorageIndexFileCommand
+ extends
BookieCommand<RegenerateInterleavedStorageIndexFileCommand.RISIFFlags> {
+
+ static final Logger LOG =
LoggerFactory.getLogger(RegenerateInterleavedStorageIndexFileCommand.class);
+
+ private static final String NAME =
"regenerate-interleaved-storage-index-file";
+ private static final String DESC =
+ "Regenerate an interleaved storage index file, from available
entrylogger " + "files.";
+ private static final String DEFAULT = "";
+
+ public RegenerateInterleavedStorageIndexFileCommand() {
+ this(new RISIFFlags());
+ }
+
+ private RegenerateInterleavedStorageIndexFileCommand(RISIFFlags flags) {
+ super(CliSpec.<RISIFFlags>newBuilder()
+ .withName(NAME)
+ .withDescription(DESC)
+ .withFlags(flags)
+ .build());
+ }
+
+ /**
+ * Flags for regenerate interleaved storage index file command.
+ */
+ @Accessors(fluent = true)
+ @Setter
+ public static class RISIFFlags extends CliFlags {
+
+ @Parameter(names = { "-p", "--password" },
+ description = "The bookie stores the password in the index file,
so we need it to regenerate."
+ + "This must match the value in the ledger
metadata.")
+ private String password = DEFAULT;
+
+ @Parameter(names = { "-b", "--b64password" },
+ description = "The password in base64 encoding, for cases where
the password is not UTF-8.")
+ private String b64Password = DEFAULT;
+
+ @Parameter(names = { "-d", "--dryrun" }, description = "Process the
entryLogger, but don't write anthing.")
+ private boolean dryRun;
+
+ @Parameter(names = { "-l", "--ledgerids" },
+ description = "Ledger(s) whose index needs to be regenerated.
Multiple can be specified, comma separated.",
+ splitter = CommaParameterSplitter.class)
+ private List<Long> ledgerIds;
+
+ }
+
+ @Override
+ public boolean apply(ServerConfiguration conf, RISIFFlags cmdFlags) {
+ try {
+ return generate(conf, cmdFlags);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ }
+
+ private boolean generate(ServerConfiguration conf, RISIFFlags flags)
throws NoSuchAlgorithmException, IOException {
+ validateFlags(flags);
+ byte[] password;
+ if (!flags.password.equals(DEFAULT)) {
+ password = flags.password.getBytes(StandardCharsets.UTF_8);
+ } else if (!flags.b64Password.equals(DEFAULT)) {
+ password = Base64.getDecoder().decode(flags.b64Password);
+ } else {
+ LOG.error("The password must be specified to regenerate the index
file");
+ return false;
+ }
+
+ Set<Long> ledgerIds =
flags.ledgerIds.stream().collect(Collectors.toSet());
+
+ LOG.info("=== Rebuilding index file for {} ===", ledgerIds);
+ ServerConfiguration serverConfiguration = new
ServerConfiguration(conf);
+ InterleavedStorageRegenerateIndexOp i = new
InterleavedStorageRegenerateIndexOp(serverConfiguration, ledgerIds,
+
password);
+ i.initiate(flags.dryRun);
+
+ LOG.info("-- Done rebuilding index file for {} --", ledgerIds);
+ return true;
+ }
+
+ private void validateFlags(RISIFFlags flags) {
+ if (flags.password == null) {
+ flags.password = DEFAULT;
+ }
+ if (flags.b64Password == null) {
+ flags.b64Password = DEFAULT;
+ }
+ }
+}
diff --git
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
index 95cd459..9e94cd2 100644
---
a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
+++
b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
@@ -34,6 +34,7 @@ import
org.apache.bookkeeper.tools.cli.commands.bookie.ReadJournalCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLedgerCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogMetadataCommand;
+import
org.apache.bookkeeper.tools.cli.commands.bookie.RegenerateInterleavedStorageIndexFileCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.SanityTestCommand;
import org.apache.bookkeeper.tools.common.BKFlags;
import org.apache.bookkeeper.tools.framework.CliCommandGroup;
@@ -66,6 +67,7 @@ public class BookieCommandGroup extends
CliCommandGroup<BKFlags> {
.addCommand(new ReadLogCommand())
.addCommand(new ReadLogMetadataCommand())
.addCommand(new LocalConsistencyCheckCommand())
+ .addCommand(new RegenerateInterleavedStorageIndexFileCommand())
.build();
public BookieCommandGroup() {
diff --git
a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/RegenerateInterleavedStorageIndexFileCommandTest.java
b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/RegenerateInterleavedStorageIndexFileCommandTest.java
new file mode 100644
index 0000000..1ade885
--- /dev/null
+++
b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/RegenerateInterleavedStorageIndexFileCommandTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookie;
+
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.bookkeeper.bookie.InterleavedStorageRegenerateIndexOp;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommandTestBase;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+/**
+ * Unit test for {@link RegenerateInterleavedStorageIndexFileCommand}.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ RegenerateInterleavedStorageIndexFileCommand.class,
InterleavedStorageRegenerateIndexOp.class,
+ ServerConfiguration.class })
+public class RegenerateInterleavedStorageIndexFileCommandTest extends
BookieCommandTestBase {
+
+ @Mock
+ private ServerConfiguration serverConfiguration;
+
+ @Mock
+ private InterleavedStorageRegenerateIndexOp op;
+
+ public RegenerateInterleavedStorageIndexFileCommandTest() {
+ super(3, 0);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+
PowerMockito.whenNew(ServerConfiguration.class).withNoArguments().thenReturn(conf);
+
PowerMockito.whenNew(ServerConfiguration.class).withArguments(eq(conf)).thenReturn(serverConfiguration);
+ }
+
+ @Test
+ public void testCommand() throws Exception {
+ String ledgerIds = "1,2,3";
+ String password = "12345";
+ Set<Long> ledgerIdsSet = Arrays.stream(ledgerIds.split(",")).map((id)
-> Long.parseLong(id))
+ .collect(Collectors.toSet());
+ byte[] bytes = password.getBytes();
+ PowerMockito.whenNew(InterleavedStorageRegenerateIndexOp.class)
+ .withArguments(eq(serverConfiguration), eq(ledgerIdsSet),
eq(bytes)).thenReturn(op);
+ PowerMockito.doNothing().when(op).initiate(anyBoolean());
+
+ RegenerateInterleavedStorageIndexFileCommand cmd = new
RegenerateInterleavedStorageIndexFileCommand();
+ Assert.assertTrue(cmd.apply(bkFlags, new String[] { "-p", password,
"-l", ledgerIds }));
+ PowerMockito.verifyNew(ServerConfiguration.class,
times(1)).withArguments(eq(conf));
+ PowerMockito.verifyNew(InterleavedStorageRegenerateIndexOp.class,
times(1))
+ .withArguments(eq(serverConfiguration), eq(ledgerIdsSet),
eq(bytes));
+ verify(op, times(1)).initiate(anyBoolean());
+ }
+}
+
