Repository: zookeeper Updated Branches: refs/heads/master 519db3f5c -> 154f9c536
ZOOKEEPER-2994: Tool required to recover log and snapshot entries with CRC errors https://issues.apache.org/jira/browse/ZOOKEEPER-2994 In the event of ZooKeeper transaction log becomes corrupted and fail CRC checks (preventing startup) we should have a mechanism to get the cluster running again. Previously we achieved this by loading the broken transaction log with a modified version of ZK with disabled CRC check and forced it to write new txn log files. It has proven that once you end up with the corrupt txn log there is no way to recover except manually modifying the crc check. That's basically why the tool is needed. It's called TxnLogToolkit, a new console application similar to LogFormatter and SnapshotFormatter, but it's intentionally separated to keep backward compatibility in the existing tools. This PR contains TXN log tool only. You probably also notice a refactoring to extract file padding logic from FileTxnLog to reuse in the new tool. Related code changes can be reviewed alone in a separate commit if preferred. Author: Andor Molnar <an...@cloudera.com> Reviewers: ph...@apache.org Closes #487 from anmolnar/ZOOKEEPER-2994 and squashes the following commits: 221760ccc [Andor Molnar] ZOOKEEPER-2994. Added documentation and startup scripts a69d7297b [Andor Molnar] ZOOKEEPER-2994. Fix findbugs warning 0b95efefd [Andor Molnar] ZOOKEEPER-2994. Fix for unit test 15fa45c68 [Andor Molnar] ZOOKEEPER-2994. Added padding, tool renamed to TxnLogToolkit, interactive mode, etc. 6a1ad0ec4 [Andor Molnar] ZOOKEEPER-2994. Refactor FileTxnLog's padding logic to separate class for reusability 0d089ccdd [Andor Molnar] ZOOKEEPER-2994. Added new tool TxnLogTool for txn log file recovery Change-Id: I7560362633a7bc919ae6d3ca7e3588e196a1919c Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/154f9c53 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/154f9c53 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/154f9c53 Branch: refs/heads/master Commit: 154f9c536f7c696b774361937c6b4b08603d16cf Parents: 519db3f Author: Andor Molnar <an...@cloudera.com> Authored: Mon Apr 23 15:20:26 2018 -0700 Committer: Patrick Hunt <ph...@apache.org> Committed: Mon Apr 23 15:20:26 2018 -0700 ---------------------------------------------------------------------- bin/zkTxnLogToolkit.cmd | 24 ++ bin/zkTxnLogToolkit.sh | 38 +++ docs/zookeeperAdmin.html | 62 ++++ docs/zookeeperAdmin.pdf | Bin 98317 -> 101979 bytes .../content/xdocs/zookeeperAdmin.xml | 70 ++++ .../server/persistence/FilePadding.java | 105 ++++++ .../server/persistence/FileTxnLog.java | 70 +--- .../server/persistence/TxnLogToolkit.java | 319 +++++++++++++++++++ src/java/test/data/invalidsnap/version-2/log.42 | Bin 0 -> 184 bytes .../server/persistence/FileTxnLogTest.java | 12 +- .../server/persistence/TxnLogToolkitTest.java | 155 +++++++++ .../org/apache/zookeeper/test/ClientBase.java | 3 +- 12 files changed, 784 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/bin/zkTxnLogToolkit.cmd ---------------------------------------------------------------------- diff --git a/bin/zkTxnLogToolkit.cmd b/bin/zkTxnLogToolkit.cmd new file mode 100755 index 0000000..362dc44 --- /dev/null +++ b/bin/zkTxnLogToolkit.cmd @@ -0,0 +1,24 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal +call "%~dp0zkEnv.cmd" + +set ZOOMAIN=org.apache.zookeeper.server.persistence.TxnLogToolkit +call %JAVA% -cp "%CLASSPATH%" %ZOOMAIN% %* + +endlocal + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/bin/zkTxnLogToolkit.sh ---------------------------------------------------------------------- diff --git a/bin/zkTxnLogToolkit.sh b/bin/zkTxnLogToolkit.sh new file mode 100755 index 0000000..8beed20 --- /dev/null +++ b/bin/zkTxnLogToolkit.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# If this scripted is run out of /usr/bin or some other system bin directory +# it should be linked to and not copied. Things like java jar files are found +# relative to the canonical path of this script. +# + +# use POSIX interface, symlink is followed automatically +ZOOBIN="${BASH_SOURCE-$0}" +ZOOBIN="$(dirname "${ZOOBIN}")" +ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)" + +if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then + . "$ZOOBINDIR"/../libexec/zkEnv.sh +else + . "$ZOOBINDIR"/zkEnv.sh +fi + +"$JAVA" -cp "$CLASSPATH" $JVMFLAGS \ + org.apache.zookeeper.server.persistence.TxnLogToolkit "$@" + + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/docs/zookeeperAdmin.html ---------------------------------------------------------------------- diff --git a/docs/zookeeperAdmin.html b/docs/zookeeperAdmin.html index 6647374..e03db49 100644 --- a/docs/zookeeperAdmin.html +++ b/docs/zookeeperAdmin.html @@ -324,6 +324,9 @@ document.write("Last Published: " + document.lastModified); <li> <a href="#sc_filemanagement">File Management</a> </li> +<li> +<a href="#Recovery+-+TxnLogToolkit">Recovery - TxnLogToolkit</a> +</li> </ul> </li> <li> @@ -2513,6 +2516,65 @@ server.3=zoo3:2888:3888</pre> </div> </div> +<a name="Recovery+-+TxnLogToolkit"></a> +<h4>Recovery - TxnLogToolkit</h4> +<p>TxnLogToolkit is a command line tool shipped with ZooKeeper which + is capable of recovering transaction log entries with broken CRC.</p> +<p>Running it without any command line parameters or with the "-h,--help" + argument, it outputs the following help page:</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh + + usage: TxnLogToolkit [-dhrv] txn_log_file_name + -d,--dump Dump mode. Dump all entries of the log file. (this is the default) + -h,--help Print help message + -r,--recover Recovery mode. Re-calculate CRC for broken entries. + -v,--verbose Be verbose in recovery mode: print all entries, not just fixed ones. + -y,--yes Non-interactive mode: repair all CRC errors without asking + </pre> +<p>The default behaviour is safe: it dumps the entries of the given + transaction log file to the screen: (same as using '-d,--dump' parameter)</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + 4/5/18 2:15:58 PM CEST session 0x16295bafcc40000 cxid 0x0 zxid 0x100000001 createSession 30000 + <strong>CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null</strong> + 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + 4/5/18 2:16:12 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x100000003 createSession 30000 + 4/5/18 2:17:34 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x200000001 closeSession null + 4/5/18 2:17:34 PM CEST session 0x16295bd23720000 cxid 0x0 zxid 0x200000002 createSession 30000 + 4/5/18 2:18:02 PM CEST session 0x16295bd23720000 cxid 0x2 zxid 0x200000003 create '/andor,#626262,v{s{31,s{'world,'anyone}}},F,1 + EOF reached after 6 txns. + </pre> +<p>There's a CRC error in the 2nd entry of the above transaction log file. In <strong>dump</strong> + mode, the toolkit only prints this information to the screen without touching the original file. In + <strong>recovery</strong> mode (-r,--recover flag) the original file still remains + untouched and all transactions will be copied over to a new txn log file with ".fixed" suffix. It recalculates + CRC values and copies the calculated value, if it doesn't match the original txn entry. + By default, the tool works interactively: it asks for confirmation whenever CRC error encountered.</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? + </pre> +<p>Answering <strong>Yes</strong> means the newly calculated CRC value will be outputted + to the new file. <strong>No</strong> means that the original CRC value will be copied over. + <strong>Abort</strong> will abort the entire operation and exits. + (In this case the ".fixed" will not be deleted and left in a half-complete state: contains only entries which + have already been processed or only the header if the operation was aborted at the first entry.)</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? y + EOF reached after 6 txns. + Recovery file log.100000001.fixed has been written with 1 fixed CRC error(s) + </pre> +<p>The default behaviour of recovery is to be silent: only entries with CRC error get printed to the screen. + One can turn on verbose mode with the -v,--verbose parameter to see all records. + Interactive mode can be turned off with the -y,--yes parameter. In this case all CRC errors will be fixed + in the new transaction file.</p> <a name="sc_commonProblems"></a> <h3 class="h4">Things to Avoid</h3> <p>Here are some common problems you can avoid by configuring http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/docs/zookeeperAdmin.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperAdmin.pdf b/docs/zookeeperAdmin.pdf index e2d01f4..7c9aec4 100644 Binary files a/docs/zookeeperAdmin.pdf and b/docs/zookeeperAdmin.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml ---------------------------------------------------------------------- diff --git a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml index 983c9ce..c59ea44 100644 --- a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml +++ b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml @@ -2159,6 +2159,76 @@ server.3=zoo3:2888:3888</programlisting> individual settings in which it is being deployed. </para> </note> </section> + + <section> + <title>Recovery - TxnLogToolkit</title> + + <para>TxnLogToolkit is a command line tool shipped with ZooKeeper which + is capable of recovering transaction log entries with broken CRC.</para> + <para>Running it without any command line parameters or with the "-h,--help" + argument, it outputs the following help page:</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh + + usage: TxnLogToolkit [-dhrv] txn_log_file_name + -d,--dump Dump mode. Dump all entries of the log file. (this is the default) + -h,--help Print help message + -r,--recover Recovery mode. Re-calculate CRC for broken entries. + -v,--verbose Be verbose in recovery mode: print all entries, not just fixed ones. + -y,--yes Non-interactive mode: repair all CRC errors without asking + </programlisting> + + <para>The default behaviour is safe: it dumps the entries of the given + transaction log file to the screen: (same as using '-d,--dump' parameter)</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + 4/5/18 2:15:58 PM CEST session 0x16295bafcc40000 cxid 0x0 zxid 0x100000001 createSession 30000 + <emphasis role="bold">CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null</emphasis> + 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + 4/5/18 2:16:12 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x100000003 createSession 30000 + 4/5/18 2:17:34 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x200000001 closeSession null + 4/5/18 2:17:34 PM CEST session 0x16295bd23720000 cxid 0x0 zxid 0x200000002 createSession 30000 + 4/5/18 2:18:02 PM CEST session 0x16295bd23720000 cxid 0x2 zxid 0x200000003 create '/andor,#626262,v{s{31,s{'world,'anyone}}},F,1 + EOF reached after 6 txns. + </programlisting> + + <para>There's a CRC error in the 2nd entry of the above transaction log file. In <emphasis role="bold">dump</emphasis> + mode, the toolkit only prints this information to the screen without touching the original file. In + <emphasis role="bold">recovery</emphasis> mode (-r,--recover flag) the original file still remains + untouched and all transactions will be copied over to a new txn log file with ".fixed" suffix. It recalculates + CRC values and copies the calculated value, if it doesn't match the original txn entry. + By default, the tool works interactively: it asks for confirmation whenever CRC error encountered.</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? + </programlisting> + + <para>Answering <emphasis role="bold">Yes</emphasis> means the newly calculated CRC value will be outputted + to the new file. <emphasis role="bold">No</emphasis> means that the original CRC value will be copied over. + <emphasis role="bold">Abort</emphasis> will abort the entire operation and exits. + (In this case the ".fixed" will not be deleted and left in a half-complete state: contains only entries which + have already been processed or only the header if the operation was aborted at the first entry.)</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? y + EOF reached after 6 txns. + Recovery file log.100000001.fixed has been written with 1 fixed CRC error(s) + </programlisting> + + <para>The default behaviour of recovery is to be silent: only entries with CRC error get printed to the screen. + One can turn on verbose mode with the -v,--verbose parameter to see all records. + Interactive mode can be turned off with the -y,--yes parameter. In this case all CRC errors will be fixed + in the new transaction file.</para> + </section> </section> <section id="sc_commonProblems"> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java b/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java new file mode 100644 index 0000000..c4052e9 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +public class FilePadding { + private static final Logger LOG; + private static long preAllocSize = 65536 * 1024; + private static final ByteBuffer fill = ByteBuffer.allocateDirect(1); + + static { + LOG = LoggerFactory.getLogger(FileTxnLog.class); + + String size = System.getProperty("zookeeper.preAllocSize"); + if (size != null) { + try { + preAllocSize = Long.parseLong(size) * 1024; + } catch (NumberFormatException e) { + LOG.warn(size + " is not a valid value for preAllocSize"); + } + } + } + + private long currentSize; + + /** + * method to allow setting preallocate size + * of log file to pad the file. + * + * @param size the size to set to in bytes + */ + public static void setPreallocSize(long size) { + preAllocSize = size; + } + + public void setCurrentSize(long currentSize) { + this.currentSize = currentSize; + } + + /** + * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position + * + * @param fileChannel the fileChannel of the file to be padded + * @throws IOException + */ + long padFile(FileChannel fileChannel) throws IOException { + long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); + if (currentSize != newFileSize) { + fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); + currentSize = newFileSize; + } + return currentSize; + } + + /** + * Calculates a new file size with padding. We only return a new size if + * the current file position is sufficiently close (less than 4K) to end of + * file and preAllocSize is > 0. + * + * @param position the point in the file we have written to + * @param fileSize application keeps track of the current file size + * @param preAllocSize how many bytes to pad + * @return the new file size. It can be the same as fileSize if no + * padding was done. + * @throws IOException + */ + // VisibleForTesting + public static long calculateFileSizeWithPadding(long position, long fileSize, long preAllocSize) { + // If preAllocSize is positive and we are within 4KB of the known end of the file calculate a new file size + if (preAllocSize > 0 && position + 4096 >= fileSize) { + // If we have written more than we have previously preallocated we need to make sure the new + // file size is larger than what we already have + if (position > fileSize) { + fileSize = position + preAllocSize; + fileSize -= fileSize % preAllocSize; + } else { + fileSize += preAllocSize; + } + } + + return fileSize; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java index 72ec606..fae7f02 100644 --- a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java +++ b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java @@ -91,9 +91,6 @@ import org.slf4j.LoggerFactory; public class FileTxnLog implements TxnLog { private static final Logger LOG; - static long preAllocSize = 65536 * 1024; - private static final ByteBuffer fill = ByteBuffer.allocateDirect(1); - public final static int TXNLOG_MAGIC = ByteBuffer.wrap("ZKLG".getBytes()).getInt(); @@ -107,14 +104,6 @@ public class FileTxnLog implements TxnLog { static { LOG = LoggerFactory.getLogger(FileTxnLog.class); - String size = System.getProperty("zookeeper.preAllocSize"); - if (size != null) { - try { - preAllocSize = Long.parseLong(size) * 1024; - } catch (NumberFormatException e) { - LOG.warn(size + " is not a valid value for preAllocSize"); - } - } /** Local variable to read fsync.warningthresholdms into */ Long fsyncWarningThreshold; if ((fsyncWarningThreshold = Long.getLong("zookeeper.fsync.warningthresholdms")) == null) @@ -132,8 +121,8 @@ public class FileTxnLog implements TxnLog { long dbId; private LinkedList<FileOutputStream> streamsToFlush = new LinkedList<FileOutputStream>(); - long currentSize; File logFileWrite = null; + private FilePadding filePadding = new FilePadding(); private volatile long syncElapsedMS = -1L; @@ -147,15 +136,6 @@ public class FileTxnLog implements TxnLog { } /** - * method to allow setting preallocate size - * of log file to pad the file. - * @param size the size to set to in bytes - */ - public static void setPreallocSize(long size) { - preAllocSize = size; - } - - /** * creates a checksum algorithm to be used * @return the checksum used for this txnlog */ @@ -163,7 +143,6 @@ public class FileTxnLog implements TxnLog { return new Adler32(); } - /** * rollover the current log file to a new one. * @throws IOException @@ -221,10 +200,10 @@ public class FileTxnLog implements TxnLog { fhdr.serialize(oa, "fileheader"); // Make sure that the magic number is written before padding. logStream.flush(); - currentSize = fos.getChannel().position(); + filePadding.setCurrentSize(fos.getChannel().position()); streamsToFlush.add(fos); } - currentSize = padFile(fos.getChannel()); + filePadding.padFile(fos.getChannel()); byte[] buf = Util.marshallTxnEntry(hdr, txn); if (buf == null || buf.length == 0) { throw new IOException("Faulty serialization for header " + @@ -239,49 +218,6 @@ public class FileTxnLog implements TxnLog { } /** - * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position - * @param fileChannel the fileChannel of the file to be padded - * @throws IOException - */ - private long padFile(FileChannel fileChannel) throws IOException { - long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); - if (currentSize != newFileSize) { - fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); - currentSize = newFileSize; - } - return currentSize; - } - - /** - * Calculates a new file size with padding. We only return a new size if - * the current file position is sufficiently close (less than 4K) to end of - * file and preAllocSize is > 0. - * - * @param position the point in the file we have written to - * @param fileSize application keeps track of the current file size - * @param preAllocSize how many bytes to pad - * @return the new file size. It can be the same as fileSize if no - * padding was done. - * @throws IOException - */ - // VisibleForTesting - public static long calculateFileSizeWithPadding(long position, long fileSize, long preAllocSize) { - // If preAllocSize is positive and we are within 4KB of the known end of the file calculate a new file size - if (preAllocSize > 0 && position + 4096 >= fileSize) { - // If we have written more than we have previously preallocated we need to make sure the new - // file size is larger than what we already have - if (position > fileSize){ - fileSize = position + preAllocSize; - fileSize -= fileSize % preAllocSize; - } else { - fileSize += preAllocSize; - } - } - - return fileSize; - } - - /** * Find the log file that starts at, or just before, the snapshot. Return * this and all subsequent logs. Results are ordered by zxid of file, * ascending order. http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java new file mode 100644 index 0000000..c6703ea --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.jute.BinaryInputArchive; +import org.apache.jute.BinaryOutputArchive; +import org.apache.jute.Record; +import org.apache.zookeeper.server.TraceFormatter; +import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.txn.TxnHeader; + +import java.io.Closeable; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.text.DateFormat; +import java.util.Date; +import java.util.Scanner; +import java.util.zip.Adler32; +import java.util.zip.Checksum; + +import static org.apache.zookeeper.server.persistence.FileTxnLog.TXNLOG_MAGIC; + +public class TxnLogToolkit implements Closeable { + + static class TxnLogToolkitException extends Exception { + private static final long serialVersionUID = 1L; + private int exitCode; + + TxnLogToolkitException(int exitCode, String message, Object... params) { + super(String.format(message, params)); + this.exitCode = exitCode; + } + + int getExitCode() { + return exitCode; + } + } + + static class TxnLogToolkitParseException extends TxnLogToolkitException { + private static final long serialVersionUID = 1L; + private Options options; + + TxnLogToolkitParseException(Options options, int exitCode, String message, Object... params) { + super(exitCode, message, params); + this.options = options; + } + + Options getOptions() { + return options; + } + } + + private File txnLogFile; + private boolean recoveryMode = false; + private boolean verbose = false; + private FileInputStream txnFis; + private BinaryInputArchive logStream; + + // Recovery mode + private int crcFixed = 0; + private FileOutputStream recoveryFos; + private BinaryOutputArchive recoveryOa; + private File recoveryLogFile; + private FilePadding filePadding = new FilePadding(); + private boolean force = false; + + /** + * @param args Command line arguments + */ + public static void main(String[] args) throws Exception { + try (final TxnLogToolkit lt = parseCommandLine(args)) { + lt.dump(new InputStreamReader(System.in)); + lt.printStat(); + } catch (TxnLogToolkitParseException e) { + System.err.println(e.getMessage() + "\n"); + printHelpAndExit(e.getExitCode(), e.getOptions()); + } catch (TxnLogToolkitException e) { + System.err.println(e.getMessage()); + System.exit(e.getExitCode()); + } + } + + public TxnLogToolkit(boolean recoveryMode, boolean verbose, String txnLogFileName, boolean force) + throws FileNotFoundException, TxnLogToolkitException { + this.recoveryMode = recoveryMode; + this.verbose = verbose; + this.force = force; + txnLogFile = new File(txnLogFileName); + if (!txnLogFile.exists() || !txnLogFile.canRead()) { + throw new TxnLogToolkitException(1, "File doesn't exist or not readable: %s", txnLogFile); + } + if (recoveryMode) { + recoveryLogFile = new File(txnLogFile.toString() + ".fixed"); + if (recoveryLogFile.exists()) { + throw new TxnLogToolkitException(1, "Recovery file %s already exists or not writable", recoveryLogFile); + } + } + + openTxnLogFile(); + if (recoveryMode) { + openRecoveryFile(); + } + } + + public void dump(Reader input) throws Exception { + crcFixed = 0; + + FileHeader fhdr = new FileHeader(); + fhdr.deserialize(logStream, "fileheader"); + if (fhdr.getMagic() != TXNLOG_MAGIC) { + throw new TxnLogToolkitException(2, "Invalid magic number for %s", txnLogFile.getName()); + } + System.out.println("ZooKeeper Transactional Log File with dbid " + + fhdr.getDbid() + " txnlog format version " + + fhdr.getVersion()); + + if (recoveryMode) { + fhdr.serialize(recoveryOa, "fileheader"); + recoveryFos.flush(); + filePadding.setCurrentSize(recoveryFos.getChannel().position()); + } + + int count = 0; + while (true) { + long crcValue; + byte[] bytes; + try { + crcValue = logStream.readLong("crcvalue"); + bytes = logStream.readBuffer("txnEntry"); + } catch (EOFException e) { + System.out.println("EOF reached after " + count + " txns."); + return; + } + if (bytes.length == 0) { + // Since we preallocate, we define EOF to be an + // empty transaction + System.out.println("EOF reached after " + count + " txns."); + return; + } + Checksum crc = new Adler32(); + crc.update(bytes, 0, bytes.length); + if (crcValue != crc.getValue()) { + if (recoveryMode) { + if (!force) { + printTxn(bytes, "CRC ERROR"); + if (askForFix(input)) { + crcValue = crc.getValue(); + ++crcFixed; + } + } else { + crcValue = crc.getValue(); + printTxn(bytes, "CRC FIXED"); + ++crcFixed; + } + } else { + printTxn(bytes, "CRC ERROR"); + } + } + if (!recoveryMode || verbose) { + printTxn(bytes); + } + if (logStream.readByte("EOR") != 'B') { + throw new TxnLogToolkitException(1, "Last transaction was partial."); + } + if (recoveryMode) { + filePadding.padFile(recoveryFos.getChannel()); + recoveryOa.writeLong(crcValue, "crcvalue"); + recoveryOa.writeBuffer(bytes, "txnEntry"); + recoveryOa.writeByte((byte)'B', "EOR"); + } + count++; + } + } + + private boolean askForFix(Reader input) throws TxnLogToolkitException { + try (Scanner scanner = new Scanner(input)) { + while (true) { + System.out.print("Would you like to fix it (Yes/No/Abort) ? "); + char answer = Character.toUpperCase(scanner.next().charAt(0)); + switch (answer) { + case 'Y': + return true; + case 'N': + return false; + case 'A': + throw new TxnLogToolkitException(0, "Recovery aborted."); + } + } + } + } + + private void printTxn(byte[] bytes) throws IOException { + printTxn(bytes, ""); + } + + private void printTxn(byte[] bytes, String prefix) throws IOException { + TxnHeader hdr = new TxnHeader(); + Record txn = SerializeUtils.deserializeTxn(bytes, hdr); + String txns = String.format("%s session 0x%s cxid 0x%s zxid 0x%s %s %s", + DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.LONG).format(new Date(hdr.getTime())), + Long.toHexString(hdr.getClientId()), + Long.toHexString(hdr.getCxid()), + Long.toHexString(hdr.getZxid()), + TraceFormatter.op2String(hdr.getType()), + txn); + if (prefix != null && !"".equals(prefix.trim())) { + System.out.print(prefix + " - "); + } + if (txns.endsWith("\n")) { + System.out.print(txns); + } else { + System.out.println(txns); + } + } + + private void openTxnLogFile() throws FileNotFoundException { + txnFis = new FileInputStream(txnLogFile); + logStream = BinaryInputArchive.getArchive(txnFis); + } + + private void closeTxnLogFile() throws IOException { + if (txnFis != null) { + txnFis.close(); + } + } + + private void openRecoveryFile() throws FileNotFoundException { + recoveryFos = new FileOutputStream(recoveryLogFile); + recoveryOa = BinaryOutputArchive.getArchive(recoveryFos); + } + + private void closeRecoveryFile() throws IOException { + if (recoveryFos != null) { + recoveryFos.close(); + } + } + + private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolkitException, FileNotFoundException { + CommandLineParser parser = new PosixParser(); + Options options = new Options(); + + Option helpOpt = new Option("h", "help", false, "Print help message"); + options.addOption(helpOpt); + + Option recoverOpt = new Option("r", "recover", false, "Recovery mode. Re-calculate CRC for broken entries."); + options.addOption(recoverOpt); + + Option quietOpt = new Option("v", "verbose", false, "Be verbose in recovery mode: print all entries, not just fixed ones."); + options.addOption(quietOpt); + + Option dumpOpt = new Option("d", "dump", false, "Dump mode. Dump all entries of the log file. (this is the default)"); + options.addOption(dumpOpt); + + Option forceOpt = new Option("y", "yes", false, "Non-interactive mode: repair all CRC errors without asking"); + options.addOption(forceOpt); + + try { + CommandLine cli = parser.parse(options, args); + if (cli.hasOption("help")) { + printHelpAndExit(0, options); + } + if (cli.getArgs().length < 1) { + printHelpAndExit(1, options); + } + return new TxnLogToolkit(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0], cli.hasOption("yes")); + } catch (ParseException e) { + throw new TxnLogToolkitParseException(options, 1, e.getMessage()); + } + } + + private static void printHelpAndExit(int exitCode, Options options) { + HelpFormatter help = new HelpFormatter(); + help.printHelp(120,"TxnLogToolkit [-dhrv] <txn_log_file_name>", "", options, ""); + System.exit(exitCode); + } + + private void printStat() { + if (recoveryMode) { + System.out.printf("Recovery file %s has been written with %d fixed CRC error(s)%n", recoveryLogFile, crcFixed); + } + } + + @Override + public void close() throws IOException { + if (recoveryMode) { + closeRecoveryFile(); + } + closeTxnLogFile(); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/test/data/invalidsnap/version-2/log.42 ---------------------------------------------------------------------- diff --git a/src/java/test/data/invalidsnap/version-2/log.42 b/src/java/test/data/invalidsnap/version-2/log.42 new file mode 100644 index 0000000..5385be5 Binary files /dev/null and b/src/java/test/data/invalidsnap/version-2/log.42 differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java index 5f54d0e..97cbc37 100644 --- a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java +++ b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java @@ -39,27 +39,27 @@ public class FileTxnLogTest extends ZKTestCase { @Test public void testInvalidPreallocSize() { Assert.assertEquals("file should not be padded", - 10 * KB, FileTxnLog.calculateFileSizeWithPadding(7 * KB, 10 * KB, 0)); + 10 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, 0)); Assert.assertEquals("file should not be padded", - 10 * KB, FileTxnLog.calculateFileSizeWithPadding(7 * KB, 10 * KB, -1)); + 10 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, -1)); } @Test public void testCalculateFileSizeWithPaddingWhenNotToCurrentSize() { Assert.assertEquals("file should not be padded", - 10 * KB, FileTxnLog.calculateFileSizeWithPadding(5 * KB, 10 * KB, 10 * KB)); + 10 * KB, FilePadding.calculateFileSizeWithPadding(5 * KB, 10 * KB, 10 * KB)); } @Test public void testCalculateFileSizeWithPaddingWhenCloseToCurrentSize() { Assert.assertEquals("file should be padded an additional 10 KB", - 20 * KB, FileTxnLog.calculateFileSizeWithPadding(7 * KB, 10 * KB, 10 * KB)); + 20 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, 10 * KB)); } @Test public void testFileSizeGreaterThanPosition() { Assert.assertEquals("file should be padded to 40 KB", - 40 * KB, FileTxnLog.calculateFileSizeWithPadding(31 * KB, 10 * KB, 10 * KB)); + 40 * KB, FilePadding.calculateFileSizeWithPadding(31 * KB, 10 * KB, 10 * KB)); } @Test @@ -69,7 +69,7 @@ public class FileTxnLogTest extends ZKTestCase { // Set a small preAllocSize (.5 MB) final int preAllocSize = 500 * KB; - fileTxnLog.setPreallocSize(preAllocSize); + FilePadding.setPreallocSize(preAllocSize); // Create dummy txn larger than preAllocSize // Since the file padding inserts a 0, we will fill the data with 0xff to ensure we corrupt the data if we put the 0 in the data http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java new file mode 100644 index 0000000..dd34fa5 --- /dev/null +++ b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.apache.commons.io.FileUtils; +import org.apache.zookeeper.test.ClientBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.io.StringReader; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + + +public class TxnLogToolkitTest { + private static final File testData = new File( + System.getProperty("test.data.dir", "build/test/data")); + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private File mySnapDir; + + @Before + public void setUp() throws IOException { + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + File snapDir = new File(testData, "invalidsnap"); + mySnapDir = ClientBase.createTmpDir(); + FileUtils.copyDirectory(snapDir, mySnapDir); + } + + @After + public void tearDown() throws IOException { + System.setOut(System.out); + System.setErr(System.err); + mySnapDir.setWritable(true); + FileUtils.deleteDirectory(mySnapDir); + } + + @Test + public void testDumpMode() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + // no exception thrown + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitException.class) + public void testInitMissingFile() throws FileNotFoundException, TxnLogToolkit.TxnLogToolkitException { + // Arrange & Act + File logfile = new File("this_file_should_not_exists"); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitException.class) + public void testInitWithRecoveryFileExists() throws IOException, TxnLogToolkit.TxnLogToolkitException { + // Arrange & Act + File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); + File recoveryFile = new File(new File(mySnapDir, "version-2"), "log.274.fixed"); + recoveryFile.createNewFile(); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); + } + + @Test + public void testDumpWithCrcError() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + String output = outContent.toString(); + Pattern p = Pattern.compile("^CRC ERROR.*session 0x8061fac5ddeb0000 cxid 0x0 zxid 0x8800000002 createSession 30000$", Pattern.MULTILINE); + Matcher m = p.matcher(output); + assertTrue("Output doesn't indicate CRC error for the broken session id: " + output, m.find()); + } + + @Test + public void testRecoveryFixBrokenFile() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + String output = outContent.toString(); + assertThat(output, containsString("CRC FIXED")); + + // Should be able to dump the recovered logfile with no CRC error + outContent.reset(); + logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); + lt = new TxnLogToolkit(false, false, logfile.toString(), true); + lt.dump(null); + output = outContent.toString(); + assertThat(output, not(containsString("CRC ERROR"))); + } + + @Test + public void testRecoveryInteractiveMode() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), false); + + // Act + lt.dump(new StringReader("y\n")); + + // Assert + String output = outContent.toString(); + assertThat(output, containsString("CRC ERROR")); + + // Should be able to dump the recovered logfile with no CRC error + outContent.reset(); + logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); + lt = new TxnLogToolkit(false, false, logfile.toString(), true); + lt.dump(null); + output = outContent.toString(); + assertThat(output, not(containsString("CRC ERROR"))); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/154f9c53/src/java/test/org/apache/zookeeper/test/ClientBase.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/test/ClientBase.java b/src/java/test/org/apache/zookeeper/test/ClientBase.java index 6e91ad3..1cdff2d 100644 --- a/src/java/test/org/apache/zookeeper/test/ClientBase.java +++ b/src/java/test/org/apache/zookeeper/test/ClientBase.java @@ -56,6 +56,7 @@ import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ServerCnxnFactoryAccessor; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FilePadding; import org.apache.zookeeper.server.persistence.FileTxnLog; import org.apache.zookeeper.server.quorum.QuorumPeer; import org.apache.zookeeper.server.util.OSMXBean; @@ -485,7 +486,7 @@ public abstract class ClientBase extends ZKTestCase { // resulting in test Assert.failure (client timeout on first session). // set env and directly in order to handle static init/gc issues System.setProperty("zookeeper.preAllocSize", "100"); - FileTxnLog.setPreallocSize(100 * 1024); + FilePadding.setPreallocSize(100 * 1024); } protected void setUpAll() throws Exception {
