Author: fpj
Date: Fri Feb 24 17:10:20 2012
New Revision: 1293342
URL: http://svn.apache.org/viewvc?rev=1293342&view=rev
Log:
BOOKKEEPER-172: Upgrade framework for filesystem layouts (ivank via fpj)
Added:
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileSystemUpgrade.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper
zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1293342&r1=1293341&r2=1293342&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Fri Feb 24 17:10:20 2012
@@ -63,6 +63,8 @@ Trunk (unreleased changes)
BOOKKEEPER-137: Do not create Ledger index files until absolutely
necessary. (ivank)
+ BOOKKEEPER-172: Upgrade framework for filesystem layouts (ivank via fpj)
+
hedwig-server/
BOOKKEEPER-77: Add a console client for hedwig (Sijie Guo via ivank)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper
URL:
http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper?rev=1293342&r1=1293341&r2=1293342&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper Fri Feb 24
17:10:20 2012
@@ -56,6 +56,7 @@ Usage: bookkeeper <command>
where command is one of:
bookie Run a bookie server
localbookie <n> Run a test ensemble of <n> bookies locally
+ upgrade Upgrade bookie filesystem
help This help message
or command is the full name of a class with a defined main() method.
@@ -126,6 +127,8 @@ elif [ $COMMAND == "localbookie" ]; then
NUMBER=$1
shift
exec java $OPTS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER
$BOOKIE_CONF $@
+elif [ $COMMAND == "upgrade" ]; then
+ exec java $OPTS org.apache.bookkeeper.bookie.FileSystemUpgrade --conf
$BOOKIE_CONF $@
elif [ $COMMAND == "help" ]; then
bookkeeper_help;
else
Added:
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileSystemUpgrade.java
URL:
http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileSystemUpgrade.java?rev=1293342&view=auto
==============================================================================
---
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileSystemUpgrade.java
(added)
+++
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileSystemUpgrade.java
Fri Feb 24 17:10:20 2012
@@ -0,0 +1,160 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.bookkeeper.bookie;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.ParseException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Scanner;
+import java.util.NoSuchElementException;
+
+import java.net.MalformedURLException;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.commons.configuration.ConfigurationException;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Application for upgrading the bookkeeper filesystem
+ * between versions
+ */
+public class FileSystemUpgrade {
+ static Logger LOG = LoggerFactory.getLogger(FileSystemUpgrade.class);
+
+ private int detectPreviousVersion(File directory) throws IOException {
+ final AtomicBoolean oldDataExists = new AtomicBoolean(false);
+ directory.list(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ if (name.endsWith(".txn") || name.endsWith(".idx") ||
name.endsWith(".log")
+ || name.equals(Bookie.VERSION_FILENAME)) {
+ oldDataExists.set(true);
+ }
+ return true;
+ }
+ });
+ if (!oldDataExists.get()) { // no old data, so we're ok
+ return Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION;
+ }
+ File v2versionFile = new File(directory, Bookie.VERSION_FILENAME);
+ if (!v2versionFile.exists()) {
+ return 1;
+ }
+ Scanner s = new Scanner(v2versionFile);
+ try {
+ return s.nextInt();
+ } catch (NoSuchElementException nse) {
+ LOG.error("Couldn't parse version file " + v2versionFile , nse);
+ throw new IOException("Couldn't parse version file", nse);
+ } catch (IllegalStateException ise) {
+ LOG.error("Error reading file " + v2versionFile, ise);
+ throw new IOException("Error reading version file", ise);
+ } finally {
+ s.close();
+ }
+ }
+
+ public static void upgrade(ServerConfiguration conf) {
+ LOG.info("Upgrading...");
+ // noop at the moment
+ LOG.info("Done");
+ }
+
+ public static void finalizeUpgrade(ServerConfiguration conf) {
+ LOG.info("Finalizing upgrade...");
+ // noop at the moment
+ LOG.info("Done");
+ }
+
+ public static void rollback(ServerConfiguration conf) {
+ LOG.info("Rolling back upgrade...");
+ // noop at the moment
+ LOG.info("Done");
+ }
+
+ private static void printHelp(Options opts) {
+ HelpFormatter hf = new HelpFormatter();
+ hf.printHelp("FileSystemUpgrade [options]", opts);
+ }
+
+ public static void main(String[] args) throws Exception {
+ org.apache.log4j.Logger root = org.apache.log4j.Logger.getRootLogger();
+ root.addAppender(new org.apache.log4j.ConsoleAppender(
+ new org.apache.log4j.PatternLayout("%-5p
[%t]: %m%n")));
+ root.setLevel(org.apache.log4j.Level.ERROR);
+
root.getLogger(FileSystemUpgrade.class).setLevel(org.apache.log4j.Level.INFO);
+
+ final Options opts = new Options();
+ opts.addOption("c", "conf", true, "Configuration for Bookie");
+ opts.addOption("u", "upgrade", false, "Upgrade bookie directories");
+ opts.addOption("f", "finalize", false, "Finalize upgrade");
+ opts.addOption("r", "rollback", false, "Rollback upgrade");
+ opts.addOption("h", "help", false, "Print help message");
+
+ BasicParser parser = new BasicParser();
+ CommandLine cmdLine = parser.parse(opts, args);
+ if (cmdLine.hasOption("h")) {
+ printHelp(opts);
+ return;
+ }
+
+ if (!cmdLine.hasOption("c")) {
+ String err = "Cannot upgrade without configuration";
+ LOG.error(err);
+ printHelp(opts);
+ throw new IllegalArgumentException(err);
+ }
+
+ String confFile = cmdLine.getOptionValue("c");
+ ServerConfiguration conf = new ServerConfiguration();
+ try {
+ conf.loadConf(new File(confFile).toURI().toURL());
+ } catch (MalformedURLException mue) {
+ LOG.error("Could not open configuration file " + confFile, mue);
+ throw new IllegalArgumentException();
+ } catch (ConfigurationException ce) {
+ LOG.error("Invalid configuration file " + confFile, ce);
+ throw new IllegalArgumentException();
+ }
+
+ if (cmdLine.hasOption("u")) {
+ upgrade(conf);
+ } else if (cmdLine.hasOption("r")) {
+ rollback(conf);
+ } else if (cmdLine.hasOption("f")) {
+ finalizeUpgrade(conf);
+ } else {
+ String err = "Must specify -upgrade, -finalize or -rollback";
+ LOG.error(err);
+ printHelp(opts);
+ throw new IllegalArgumentException(err);
+ }
+ }
+}
\ No newline at end of file
Added:
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
URL:
http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java?rev=1293342&view=auto
==============================================================================
---
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
(added)
+++
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
Fri Feb 24 17:10:20 2012
@@ -0,0 +1,115 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.bookkeeper.bookie;
+
+import java.io.File;
+import java.io.IOException;
+
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.BufferedWriter;
+import java.io.PrintStream;
+
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+import org.apache.bookkeeper.conf.ServerConfiguration;
+
+public class UpgradeTest {
+ static String newV1JournalDirectory() throws IOException {
+ File d = File.createTempFile("bookie", "tmpdir");
+ d.delete();
+ d.mkdirs();
+ new File(d, Long.toHexString(System.currentTimeMillis()) +
".txn").createNewFile();
+ return d.getPath();
+ }
+
+ static String newV1LedgerDirectory() throws IOException {
+ File d = File.createTempFile("bookie", "tmpdir");
+ d.delete();
+ d.mkdirs();
+ new File(d, Long.toHexString(System.currentTimeMillis()) +
".log").createNewFile();
+ return d.getPath();
+ }
+
+ static void createVersion2File(String dir) throws IOException {
+ File versionFile = new File(dir, "VERSION");
+
+ FileOutputStream fos = new FileOutputStream(versionFile);
+ BufferedWriter bw = null;
+ try {
+ bw = new BufferedWriter(new OutputStreamWriter(fos));
+ bw.write(String.valueOf(2));
+ } finally {
+ if (bw != null) {
+ bw.close();
+ }
+ fos.close();
+ }
+ }
+
+ static String newV2JournalDirectory() throws IOException {
+ String d = newV1JournalDirectory();
+ createVersion2File(d);
+ return d;
+ }
+
+ static String newV2LedgerDirectory() throws IOException {
+ String d = newV1LedgerDirectory();
+ createVersion2File(d);
+ return d;
+ }
+
+ @Test
+ public void testCommandLine() throws Exception {
+ PrintStream origerr = System.err;
+ PrintStream origout = System.out;
+
+ File output = File.createTempFile("bookie", "tmpout");
+ System.setOut(new PrintStream(output));
+ System.setErr(new PrintStream(output));
+ try {
+ FileSystemUpgrade.main(new String[] { "-h" });
+ try {
+ // test without conf
+ FileSystemUpgrade.main(new String[] { "-u" });
+ fail("Should have failed");
+ } catch (IllegalArgumentException iae) {
+ assertTrue("Wrong exception " + iae.getMessage(),
+ iae.getMessage().contains("without configuration"));
+ }
+ File f = File.createTempFile("bookie", "tmpconf");
+ try {
+ // test without upgrade op
+ FileSystemUpgrade.main(new String[] { "--conf", f.getPath() });
+ fail("Should have failed");
+ } catch (IllegalArgumentException iae) {
+ assertTrue("Wrong exception " + iae.getMessage(),
+ iae.getMessage().contains("Must specify -upgrade"));
+ }
+ } finally {
+ System.setOut(origout);
+ System.setErr(origerr);
+ }
+ }
+}
\ No newline at end of file
Modified: zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile
URL:
http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile?rev=1293342&r1=1293341&r2=1293342&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile Fri Feb 24 17:10:20
2012
@@ -14,7 +14,7 @@ Notice: Licensed under the Apache Licens
h1. Abstract
-This document contains information about deploying, administering and
mantaining BookKeeper. It also discusses best practices and common problems.
+This document contains information about deploying, administering and
maintaining BookKeeper. It also discusses best practices and common problems.
h1. Running a BookKeeper instance
@@ -35,13 +35,29 @@ p. To run a bookie, we execute the follo
p. The configuration parameters can be set in
bookkeeper-server/conf/bk_server.conf.
The important parameters are:
- * @bookiePort@, Port number that the bookie listens on;
- * @zkServers@, Comma separated list of ZooKeeper servers with a hostname:port
format;
- * @journalDir@, Path for Log Device (stores bookie write-ahead log);
- * @ledgerDir@, Path for Ledger Device (stores ledger entries);
+
+* @bookiePort@, Port number that the bookie listens on;
+* @zkServers@, Comma separated list of ZooKeeper servers with a hostname:port
format;
+* @journalDir@, Path for Log Device (stores bookie write-ahead log);
+* @ledgerDir@, Path for Ledger Device (stores ledger entries);
p. Ideally, @journalDir@ and @ledgerDir@ are each in a different device. See
"BookKeeper Configuration Parameters":./bookkeeperConfigParams.html for a full
list of configuration parameters.
+h3. Upgrading
+
+From time to time, we may make changes to the filesystem layout of the bookie,
which are incompatible with previous versions of bookkeeper and require that
directories used with previous versions are upgraded. If you upgrade your
bookkeeper software, and an upgrade is required, then the bookie will fail to
start and print an error to the effect that an upgrade is required.
+
+BookKeeper provides a utility for doing the upgrade.
+@bookkeeper-server/bin/bookkeeper upgrade@
+
+The upgrade application takes 3 possible switches, @--upgrade@, @--rollback@
or @--finalize@. A normal upgrade process looks like.
+
+# @bookkeeper-server/bin/bookkeeper upgrade --upgrade@
+# @bookkeeper-server/bin/bookkeeper bookie@
+# Check everything is working. Kill bookie, ^C
+# If everything is ok, @bookkeeper-server/bin/bookkeeper upgrade --finalize@
+# Start bookie again @bookkeeper-server/bin/bookkeeper bookie@
+# If something is amiss, you can roll back the upgrade
@bookkeeper-server/bin/bookkeeper upgrade --rollback@
h3. Logging
