ACCUMULO-2770 Add utility to read local WAL Forward port the upgrade code from 1.5.x into a separate utility so that a user can still read local WALs if they happen to be around.
Recreate LogFile{Key,Value} in their old packages so that we can still read the old sequence files that have embedded class names. Modify the utility to optionally accept values as command line options instead of searching around in the configuration settings. Testability! Add a WAL file explicitly generated under 1.4 for testing purposes. Add documentation to the troubleshooting guide to describe usage. Recover your logs After something has gone wrong Watch your flowers bloom commit 2db5ce6186e32c451328154b024951cc5090505f Author: Eric C. Newton <e...@apache.org> Date: Tue Jun 5 13:18:22 2012 +0000 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/366aef1b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/366aef1b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/366aef1b Branch: refs/heads/1.6.1-SNAPSHOT Commit: 366aef1b9f970a3a3e6d5034c336fd17695920fa Parents: b33cb73 Author: Mike Drob <md...@cloudera.com> Authored: Wed May 14 00:11:21 2014 -0400 Committer: Mike Drob <md...@cloudera.com> Committed: Wed May 14 10:36:18 2014 -0400 ---------------------------------------------------------------------- .../org/apache/accumulo/core/conf/Property.java | 8 +- .../chapters/troubleshooting.tex | 13 ++ server/tserver/pom.xml | 5 + .../accumulo/server/logger/LogFileKey.java | 25 +++ .../accumulo/server/logger/LogFileValue.java | 25 +++ .../accumulo/tserver/log/LocalWALRecovery.java | 176 +++++++++++++++++++ .../tserver/log/LocalWALRecoveryTest.java | 108 ++++++++++++ .../550e8400-e29b-41d4-a716-446655440000 | Bin 0 -> 91784 bytes 8 files changed, 356 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 60969be..47d1817 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -268,10 +268,10 @@ public enum Property { // properties that are specific to logger server behavior LOGGER_PREFIX("logger.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the write-ahead logger servers"), - LOGGER_DIR("logger.dir.walog", "walogs", PropertyType.PATH, - "The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local filesystem. In 1.5 write-ahead logs are " - + "stored in DFS. When 1.5 is started for the first time it will copy any 1.4 write ahead logs into DFS. It is possible to specify a " - + "comma-separated list of directories."), + LOGGER_DIR("logger.dir.walog", "walogs", PropertyType.PATH, "This property is only needed if Accumulo was upgraded from a 1.4 or earlier version. " + + "In the upgrade to 1.5 this property is used to copy any earlier write ahead logs into DFS. " + + "In 1.6+, this property is used by the LocalWALRecovery utility in the event that something went wrong with that earlier upgrade. " + + "It is possible to specify a comma-separated list of directories."), // accumulo garbage collector properties GC_PREFIX("gc.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the accumulo garbage collector."), http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex ---------------------------------------------------------------------- diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex b/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex index 203fe0c..57fdf13 100644 --- a/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex +++ b/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex @@ -764,6 +764,19 @@ A. The ``importdirectory`` shell command can be used to import RFiles from the o but extreme care should go into the decision to do this as it may result in reintroduction of stale data or the omission of new data. +\subsection{Upgrade Issues} +Q. I upgraded from 1.4 to 1.5 to 1.6 but still have some WAL files on local disk. Do I have any way to recover them? + +A. Yes, you can recover them by running the LocalWALRecovery utility on each node that needs recovery performed. The utility +will default to using the directory specified by \begin{verbatim}logger.dir.walog\end{verbatim} in your configuration, or can be +overriden by using the \begin{verbatim}--local-wal-directories\end{verbatim} option on the tool. It can be invoked as follows: + +\small +\begin{verbatim} +$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.tserver.log.LocalWALRecovery +\end{verbatim} +\normalsize + \section{File Naming Conventions} Q. Why are files named like they are? Why do some start with ``C'' and others with ``F''? http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/server/tserver/pom.xml ---------------------------------------------------------------------- diff --git a/server/tserver/pom.xml b/server/tserver/pom.xml index f2b47d4..a431d5d 100644 --- a/server/tserver/pom.xml +++ b/server/tserver/pom.xml @@ -93,6 +93,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileKey.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileKey.java b/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileKey.java new file mode 100644 index 0000000..80e303f --- /dev/null +++ b/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileKey.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.logger; + +/** + * @deprecated Only used for recovering local WAL files. Use {@link org.apache.accumulo.tserver.logger.LogFileKey} instead. + */ +@Deprecated +public class LogFileKey extends org.apache.accumulo.tserver.logger.LogFileKey { + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileValue.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileValue.java b/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileValue.java new file mode 100644 index 0000000..06e70b9 --- /dev/null +++ b/server/tserver/src/main/java/org/apache/accumulo/server/logger/LogFileValue.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.logger; + +/** + * @deprecated Only used for recovering local WAL files. Use {@link org.apache.accumulo.tserver.logger.LogFileValue} instead. + */ +@Deprecated +public class LogFileValue extends org.apache.accumulo.tserver.logger.LogFileValue { + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java new file mode 100644 index 0000000..2adb52d --- /dev/null +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver.log; + +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.security.SecurityUtil; +import org.apache.accumulo.server.ServerConstants; +import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.fs.VolumeManagerImpl; +import org.apache.accumulo.server.logger.LogFileKey; +import org.apache.accumulo.server.logger.LogFileValue; +import org.apache.accumulo.tserver.TabletServer; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.SequenceFile.Reader; +import org.apache.log4j.Logger; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.google.common.annotations.VisibleForTesting; + +/** + * This class will attempt to rewrite any local WALs to HDFS. + */ +public class LocalWALRecovery implements Runnable { + private static final Logger log = Logger.getLogger(TabletServer.class); + + public static void main(String[] args) throws IOException { + AccumuloConfiguration configuration = SiteConfiguration.getInstance(SiteConfiguration.getDefaultConfiguration()); + + LocalWALRecovery main = new LocalWALRecovery(configuration); + main.parseArgs(args); + main.run(); + } + + public final class Options { + @Parameter(names = "--delete-local", description = "Specify whether to delete the local WAL files after they have been re-written in HDFS.") + public boolean deleteLocal = false; + + @Parameter(names = "--local-wal-directories", + description = "Comma separated list of local directories containing WALs, default is set according to the logger.dir.walog property.") + public List<String> directories = getDefaultDirectories(); + + @Parameter(names = "--dfs-wal-directory", + description = "The directory that WALs will be copied into. Will default to the first configured base dir + '/wal'") + public String destination = null; + + private List<String> getDefaultDirectories() { + String property = configuration.get(Property.LOGGER_DIR); + return Arrays.asList(property.split(",")); + } + } + + private final AccumuloConfiguration configuration; + private final Options options; + + /** + * Create a WAL recovery tool for the given instance. + */ + public LocalWALRecovery(AccumuloConfiguration configuration) { + this.configuration = configuration; + this.options = new Options(); + } + + @VisibleForTesting + public void parseArgs(String... args) { + JCommander jcommander = new JCommander(); + jcommander.addObject(options); + + try { + jcommander.parse(args); + } catch (ParameterException e) { + jcommander.usage(); + } + } + + @Override + public void run() { + SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); + + try { + recoverLocalWriteAheadLogs(VolumeManagerImpl.get().getDefaultVolume().getFileSystem()); + } catch (IOException e) { + log.error("Error while recovering WAL files.", e); + } + } + + public void recoverLocalWriteAheadLogs(FileSystem fs) throws IOException { + for (String directory : options.directories) { + File localDirectory = new File(directory); + if (!localDirectory.isAbsolute()) { + localDirectory = new File(System.getenv("ACCUMULO_HOME"), directory); + } + + if (!localDirectory.isDirectory()) { + log.warn("Local walog dir " + localDirectory.getAbsolutePath() + " does not exist or is not a directory."); + continue; + } + + if (options.destination == null) { + // Defer loading the default value until now because it might require talking to zookeeper. + options.destination = ServerConstants.getWalDirs()[0]; + } + log.info("Copying WALs to " + options.destination); + + for (File file : localDirectory.listFiles()) { + String name = file.getName(); + try { + UUID.fromString(name); + } catch (IllegalArgumentException ex) { + log.info("Ignoring non-log file " + file.getAbsolutePath()); + continue; + } + + @SuppressWarnings("deprecation") + LogFileKey key = new LogFileKey(); + @SuppressWarnings("deprecation") + LogFileValue value = new LogFileValue(); + + log.info("Openning local log " + file.getAbsolutePath()); + + Reader reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(new Path(file.toURI()))); + Path tmp = new Path(options.destination + "/" + name + ".copy"); + FSDataOutputStream writer = fs.create(tmp); + while (reader.next(key, value)) { + try { + key.write(writer); + value.write(writer); + } catch (EOFException ex) { + break; + } + } + writer.close(); + reader.close(); + fs.rename(tmp, new Path(tmp.getParent(), name)); + + if (options.deleteLocal) { + if (file.delete()) { + log.info("Copied and deleted: " + name); + } else { + log.info("Failed to delete: " + name + " (but it is safe for you to delete it manually)."); + } + } else { + log.info("Safe to delete: " + name); + } + } + } + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LocalWALRecoveryTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LocalWALRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LocalWALRecoveryTest.java new file mode 100644 index 0000000..54752e9 --- /dev/null +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LocalWALRecoveryTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver.log; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; + +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.UUID; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; +import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams; +import org.apache.accumulo.tserver.logger.LogFileKey; +import org.apache.accumulo.tserver.logger.LogFileValue; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class LocalWALRecoveryTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + LocalWALRecovery recovery; + + File walTarget; + AccumuloConfiguration configuration; + + @Before + public void setUp() throws Exception { + configuration = createMock(AccumuloConfiguration.class); + expect(configuration.get(Property.LOGGER_DIR)).andReturn("src/test/resources/walog-from-14").anyTimes(); + replay(configuration); + + walTarget = folder.newFolder("wal"); + + recovery = new LocalWALRecovery(configuration); + recovery.parseArgs("--dfs-wal-directory", walTarget.getAbsolutePath()); + } + + @Test + public void testRecoverLocalWriteAheadLogs() throws IOException { + FileSystem fs = FileSystem.get(walTarget.toURI(), new Configuration()); + recovery.recoverLocalWriteAheadLogs(fs); + + assertEquals("Wrong number of WAL files recovered.", 1, walTarget.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + try { + // Filter out the CRC file + UUID.fromString(name); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + }).length); + + final Path path = new Path(walTarget.listFiles()[0].getAbsolutePath()); + final VolumeManager volumeManager = VolumeManagerImpl.getLocal(folder.getRoot().getAbsolutePath()); + + final DFSLoggerInputStreams streams = DfsLogger.readHeaderAndReturnStream(volumeManager, path, configuration); + final DataInputStream input = streams.getDecryptingInputStream(); + + final LogFileKey key = new LogFileKey(); + final LogFileValue value = new LogFileValue(); + int read = 0; + + while (true) { + try { + key.readFields(input); + value.readFields(input); + read++; + } catch (EOFException ex) { + break; + } + } + + assertEquals(104, read); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/366aef1b/server/tserver/src/test/resources/walog-from-14/550e8400-e29b-41d4-a716-446655440000 ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/resources/walog-from-14/550e8400-e29b-41d4-a716-446655440000 b/server/tserver/src/test/resources/walog-from-14/550e8400-e29b-41d4-a716-446655440000 new file mode 100644 index 0000000..cb3aca8 Binary files /dev/null and b/server/tserver/src/test/resources/walog-from-14/550e8400-e29b-41d4-a716-446655440000 differ