Repository: zookeeper Updated Branches: refs/heads/branch-3.4 eacb4e4f3 -> 126fb0f22
ZOOKEEPER-2994: Tool required to recover log and snapshot entries with CRC errors (3.4) This is the 3.4 version of https://github.com/apache/zookeeper/pull/487 phunt I've just realized that the patch must introduce a new dependency: commons-cli. Not sure if you're willing to merge it in this case. Author: Andor Molnar <[email protected]> Reviewers: [email protected] Closes #508 from anmolnar/ZOOKEEPER-2994_34 and squashes the following commits: 357ab2bb0 [Andor Molnar] ZOOKEEPER-2994. Removed dependency of commons.cli. Use custom impl instead. 3bc2e5f72 [Andor Molnar] ZOOKEEPER-2994: Tool required to recover log and snapshot entries with CRC errors Change-Id: I7def29dc338726c3eccb0a4fd4530a1ffb0f3932 Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/126fb0f2 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/126fb0f2 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/126fb0f2 Branch: refs/heads/branch-3.4 Commit: 126fb0f22d701cad58bf3123bf7d8f2219e60387 Parents: eacb4e4 Author: Andor Molnar <[email protected]> Authored: Wed Apr 25 14:41:31 2018 -0700 Committer: Patrick Hunt <[email protected]> Committed: Wed Apr 25 14:43:00 2018 -0700 ---------------------------------------------------------------------- bin/zkTxnLogToolkit.cmd | 24 ++ bin/zkTxnLogToolkit.sh | 38 +++ docs/bookkeeperConfig.pdf | Bin 13793 -> 13797 bytes docs/bookkeeperOverview.pdf | Bin 147554 -> 147558 bytes docs/bookkeeperProgrammer.pdf | Bin 24945 -> 24949 bytes docs/bookkeeperStarted.pdf | Bin 17103 -> 17107 bytes docs/bookkeeperStream.pdf | Bin 13188 -> 13192 bytes docs/index.pdf | Bin 13511 -> 13515 bytes docs/javaExample.pdf | Bin 33763 -> 33767 bytes docs/linkmap.pdf | Bin 12455 -> 12459 bytes docs/recipes.pdf | Bin 31017 -> 31021 bytes docs/zookeeperAdmin.html | 62 ++++ docs/zookeeperAdmin.pdf | Bin 82373 -> 86043 bytes docs/zookeeperHierarchicalQuorums.pdf | Bin 6650 -> 6654 bytes docs/zookeeperInternals.pdf | Bin 48806 -> 48810 bytes docs/zookeeperJMX.pdf | Bin 16471 -> 16475 bytes docs/zookeeperObservers.pdf | Bin 12867 -> 12871 bytes docs/zookeeperOver.pdf | Bin 302479 -> 302472 bytes docs/zookeeperProgrammers.pdf | Bin 134096 -> 134100 bytes docs/zookeeperQuotas.pdf | Bin 11180 -> 11184 bytes docs/zookeeperStarted.pdf | Bin 28069 -> 28073 bytes docs/zookeeperTutorial.pdf | Bin 34208 -> 34212 bytes .../content/xdocs/zookeeperAdmin.xml | 70 +++++ .../apache/zookeeper/server/TraceFormatter.java | 2 +- .../server/persistence/FilePadding.java | 105 +++++++ .../server/persistence/FileTxnLog.java | 90 +----- .../server/persistence/TxnLogToolkit.java | 280 +++++++++++++++++++ .../persistence/TxnLogToolkitCliParser.java | 101 +++++++ src/java/test/data/invalidsnap/version-2/log.42 | Bin 0 -> 184 bytes .../server/persistence/FileTxnLogTest.java | 12 +- .../persistence/TxnLogToolkitCliParserTest.java | 110 ++++++++ .../server/persistence/TxnLogToolkitTest.java | 155 ++++++++++ .../org/apache/zookeeper/test/ClientBase.java | 3 +- 33 files changed, 967 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/bin/zkTxnLogToolkit.cmd ---------------------------------------------------------------------- diff --git a/bin/zkTxnLogToolkit.cmd b/bin/zkTxnLogToolkit.cmd new file mode 100755 index 0000000..362dc44 --- /dev/null +++ b/bin/zkTxnLogToolkit.cmd @@ -0,0 +1,24 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal +call "%~dp0zkEnv.cmd" + +set ZOOMAIN=org.apache.zookeeper.server.persistence.TxnLogToolkit +call %JAVA% -cp "%CLASSPATH%" %ZOOMAIN% %* + +endlocal + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/bin/zkTxnLogToolkit.sh ---------------------------------------------------------------------- diff --git a/bin/zkTxnLogToolkit.sh b/bin/zkTxnLogToolkit.sh new file mode 100755 index 0000000..8beed20 --- /dev/null +++ b/bin/zkTxnLogToolkit.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# If this scripted is run out of /usr/bin or some other system bin directory +# it should be linked to and not copied. Things like java jar files are found +# relative to the canonical path of this script. +# + +# use POSIX interface, symlink is followed automatically +ZOOBIN="${BASH_SOURCE-$0}" +ZOOBIN="$(dirname "${ZOOBIN}")" +ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)" + +if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then + . "$ZOOBINDIR"/../libexec/zkEnv.sh +else + . "$ZOOBINDIR"/zkEnv.sh +fi + +"$JAVA" -cp "$CLASSPATH" $JVMFLAGS \ + org.apache.zookeeper.server.persistence.TxnLogToolkit "$@" + + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/bookkeeperConfig.pdf ---------------------------------------------------------------------- diff --git a/docs/bookkeeperConfig.pdf b/docs/bookkeeperConfig.pdf index c62f493..07b8b9c 100644 Binary files a/docs/bookkeeperConfig.pdf and b/docs/bookkeeperConfig.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/bookkeeperOverview.pdf ---------------------------------------------------------------------- diff --git a/docs/bookkeeperOverview.pdf b/docs/bookkeeperOverview.pdf index 25880ad..58c3b81 100644 Binary files a/docs/bookkeeperOverview.pdf and b/docs/bookkeeperOverview.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/bookkeeperProgrammer.pdf ---------------------------------------------------------------------- diff --git a/docs/bookkeeperProgrammer.pdf b/docs/bookkeeperProgrammer.pdf index dbf3cb6..efe127b 100644 Binary files a/docs/bookkeeperProgrammer.pdf and b/docs/bookkeeperProgrammer.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/bookkeeperStarted.pdf ---------------------------------------------------------------------- diff --git a/docs/bookkeeperStarted.pdf b/docs/bookkeeperStarted.pdf index 483d9f7..bca40a5 100644 Binary files a/docs/bookkeeperStarted.pdf and b/docs/bookkeeperStarted.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/bookkeeperStream.pdf ---------------------------------------------------------------------- diff --git a/docs/bookkeeperStream.pdf b/docs/bookkeeperStream.pdf index 5b5ca78..7d602dd 100644 Binary files a/docs/bookkeeperStream.pdf and b/docs/bookkeeperStream.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/index.pdf ---------------------------------------------------------------------- diff --git a/docs/index.pdf b/docs/index.pdf index bd53a29..de75668 100644 Binary files a/docs/index.pdf and b/docs/index.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/javaExample.pdf ---------------------------------------------------------------------- diff --git a/docs/javaExample.pdf b/docs/javaExample.pdf index f34daa4..a5a2e74 100644 Binary files a/docs/javaExample.pdf and b/docs/javaExample.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/linkmap.pdf ---------------------------------------------------------------------- diff --git a/docs/linkmap.pdf b/docs/linkmap.pdf index 0f8bcad..d756b22 100644 Binary files a/docs/linkmap.pdf and b/docs/linkmap.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/recipes.pdf ---------------------------------------------------------------------- diff --git a/docs/recipes.pdf b/docs/recipes.pdf index 5aab9ad..3c4cb9e 100644 Binary files a/docs/recipes.pdf and b/docs/recipes.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperAdmin.html ---------------------------------------------------------------------- diff --git a/docs/zookeeperAdmin.html b/docs/zookeeperAdmin.html index f8166f9..9f00dba 100644 --- a/docs/zookeeperAdmin.html +++ b/docs/zookeeperAdmin.html @@ -316,6 +316,9 @@ document.write("Last Published: " + document.lastModified); <li> <a href="#sc_filemanagement">File Management</a> </li> +<li> +<a href="#Recovery+-+TxnLogToolkit">Recovery - TxnLogToolkit</a> +</li> </ul> </li> <li> @@ -2063,6 +2066,65 @@ imok </div> </div> +<a name="Recovery+-+TxnLogToolkit"></a> +<h4>Recovery - TxnLogToolkit</h4> +<p>TxnLogToolkit is a command line tool shipped with ZooKeeper which + is capable of recovering transaction log entries with broken CRC.</p> +<p>Running it without any command line parameters or with the "-h,--help" + argument, it outputs the following help page:</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh + + usage: TxnLogToolkit [-dhrv] txn_log_file_name + -d,--dump Dump mode. Dump all entries of the log file. (this is the default) + -h,--help Print help message + -r,--recover Recovery mode. Re-calculate CRC for broken entries. + -v,--verbose Be verbose in recovery mode: print all entries, not just fixed ones. + -y,--yes Non-interactive mode: repair all CRC errors without asking + </pre> +<p>The default behaviour is safe: it dumps the entries of the given + transaction log file to the screen: (same as using '-d,--dump' parameter)</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + 4/5/18 2:15:58 PM CEST session 0x16295bafcc40000 cxid 0x0 zxid 0x100000001 createSession 30000 + <strong>CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null</strong> + 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + 4/5/18 2:16:12 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x100000003 createSession 30000 + 4/5/18 2:17:34 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x200000001 closeSession null + 4/5/18 2:17:34 PM CEST session 0x16295bd23720000 cxid 0x0 zxid 0x200000002 createSession 30000 + 4/5/18 2:18:02 PM CEST session 0x16295bd23720000 cxid 0x2 zxid 0x200000003 create '/andor,#626262,v{s{31,s{'world,'anyone}}},F,1 + EOF reached after 6 txns. + </pre> +<p>There's a CRC error in the 2nd entry of the above transaction log file. In <strong>dump</strong> + mode, the toolkit only prints this information to the screen without touching the original file. In + <strong>recovery</strong> mode (-r,--recover flag) the original file still remains + untouched and all transactions will be copied over to a new txn log file with ".fixed" suffix. It recalculates + CRC values and copies the calculated value, if it doesn't match the original txn entry. + By default, the tool works interactively: it asks for confirmation whenever CRC error encountered.</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? + </pre> +<p>Answering <strong>Yes</strong> means the newly calculated CRC value will be outputted + to the new file. <strong>No</strong> means that the original CRC value will be copied over. + <strong>Abort</strong> will abort the entire operation and exits. + (In this case the ".fixed" will not be deleted and left in a half-complete state: contains only entries which + have already been processed or only the header if the operation was aborted at the first entry.)</p> +<pre class="code"> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? y + EOF reached after 6 txns. + Recovery file log.100000001.fixed has been written with 1 fixed CRC error(s) + </pre> +<p>The default behaviour of recovery is to be silent: only entries with CRC error get printed to the screen. + One can turn on verbose mode with the -v,--verbose parameter to see all records. + Interactive mode can be turned off with the -y,--yes parameter. In this case all CRC errors will be fixed + in the new transaction file.</p> <a name="sc_commonProblems"></a> <h3 class="h4">Things to Avoid</h3> <p>Here are some common problems you can avoid by configuring http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperAdmin.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperAdmin.pdf b/docs/zookeeperAdmin.pdf index 0004649..21728be 100644 Binary files a/docs/zookeeperAdmin.pdf and b/docs/zookeeperAdmin.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperHierarchicalQuorums.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperHierarchicalQuorums.pdf b/docs/zookeeperHierarchicalQuorums.pdf index f86909f..d0db17f 100644 Binary files a/docs/zookeeperHierarchicalQuorums.pdf and b/docs/zookeeperHierarchicalQuorums.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperInternals.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperInternals.pdf b/docs/zookeeperInternals.pdf index 8bcd192..7ca1e0b 100644 Binary files a/docs/zookeeperInternals.pdf and b/docs/zookeeperInternals.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperJMX.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperJMX.pdf b/docs/zookeeperJMX.pdf index 8b74b23..56d67e1 100644 Binary files a/docs/zookeeperJMX.pdf and b/docs/zookeeperJMX.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperObservers.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperObservers.pdf b/docs/zookeeperObservers.pdf index 29ebbb0..f793f34 100644 Binary files a/docs/zookeeperObservers.pdf and b/docs/zookeeperObservers.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperOver.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperOver.pdf b/docs/zookeeperOver.pdf index 8839eef..c262197 100644 Binary files a/docs/zookeeperOver.pdf and b/docs/zookeeperOver.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperProgrammers.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperProgrammers.pdf b/docs/zookeeperProgrammers.pdf index 8448d31..b375faf 100644 Binary files a/docs/zookeeperProgrammers.pdf and b/docs/zookeeperProgrammers.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperQuotas.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperQuotas.pdf b/docs/zookeeperQuotas.pdf index 6a95063..d6894c6 100644 Binary files a/docs/zookeeperQuotas.pdf and b/docs/zookeeperQuotas.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperStarted.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperStarted.pdf b/docs/zookeeperStarted.pdf index e6e15cc..cb52a80 100644 Binary files a/docs/zookeeperStarted.pdf and b/docs/zookeeperStarted.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/docs/zookeeperTutorial.pdf ---------------------------------------------------------------------- diff --git a/docs/zookeeperTutorial.pdf b/docs/zookeeperTutorial.pdf index 70df263..543177c 100644 Binary files a/docs/zookeeperTutorial.pdf and b/docs/zookeeperTutorial.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml ---------------------------------------------------------------------- diff --git a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml index 447b90c..2f3a57f 100644 --- a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml +++ b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml @@ -1702,6 +1702,76 @@ imok individual settings in which it is being deployed. </para> </note> </section> + + <section> + <title>Recovery - TxnLogToolkit</title> + + <para>TxnLogToolkit is a command line tool shipped with ZooKeeper which + is capable of recovering transaction log entries with broken CRC.</para> + <para>Running it without any command line parameters or with the "-h,--help" + argument, it outputs the following help page:</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh + + usage: TxnLogToolkit [-dhrv] txn_log_file_name + -d,--dump Dump mode. Dump all entries of the log file. (this is the default) + -h,--help Print help message + -r,--recover Recovery mode. Re-calculate CRC for broken entries. + -v,--verbose Be verbose in recovery mode: print all entries, not just fixed ones. + -y,--yes Non-interactive mode: repair all CRC errors without asking + </programlisting> + + <para>The default behaviour is safe: it dumps the entries of the given + transaction log file to the screen: (same as using '-d,--dump' parameter)</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + 4/5/18 2:15:58 PM CEST session 0x16295bafcc40000 cxid 0x0 zxid 0x100000001 createSession 30000 + <emphasis role="bold">CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null</emphasis> + 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + 4/5/18 2:16:12 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x100000003 createSession 30000 + 4/5/18 2:17:34 PM CEST session 0x26295bafcc90000 cxid 0x0 zxid 0x200000001 closeSession null + 4/5/18 2:17:34 PM CEST session 0x16295bd23720000 cxid 0x0 zxid 0x200000002 createSession 30000 + 4/5/18 2:18:02 PM CEST session 0x16295bd23720000 cxid 0x2 zxid 0x200000003 create '/andor,#626262,v{s{31,s{'world,'anyone}}},F,1 + EOF reached after 6 txns. + </programlisting> + + <para>There's a CRC error in the 2nd entry of the above transaction log file. In <emphasis role="bold">dump</emphasis> + mode, the toolkit only prints this information to the screen without touching the original file. In + <emphasis role="bold">recovery</emphasis> mode (-r,--recover flag) the original file still remains + untouched and all transactions will be copied over to a new txn log file with ".fixed" suffix. It recalculates + CRC values and copies the calculated value, if it doesn't match the original txn entry. + By default, the tool works interactively: it asks for confirmation whenever CRC error encountered.</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? + </programlisting> + + <para>Answering <emphasis role="bold">Yes</emphasis> means the newly calculated CRC value will be outputted + to the new file. <emphasis role="bold">No</emphasis> means that the original CRC value will be copied over. + <emphasis role="bold">Abort</emphasis> will abort the entire operation and exits. + (In this case the ".fixed" will not be deleted and left in a half-complete state: contains only entries which + have already been processed or only the header if the operation was aborted at the first entry.)</para> + + <programlisting> + $ bin/zkTxnLogToolkit.sh -r log.100000001 + ZooKeeper Transactional Log File with dbid 0 txnlog format version 2 + CRC ERROR - 4/5/18 2:16:05 PM CEST session 0x16295bafcc40000 cxid 0x1 zxid 0x100000002 closeSession null + Would you like to fix it (Yes/No/Abort) ? y + EOF reached after 6 txns. + Recovery file log.100000001.fixed has been written with 1 fixed CRC error(s) + </programlisting> + + <para>The default behaviour of recovery is to be silent: only entries with CRC error get printed to the screen. + One can turn on verbose mode with the -v,--verbose parameter to see all records. + Interactive mode can be turned off with the -y,--yes parameter. In this case all CRC errors will be fixed + in the new transaction file.</para> + </section> </section> <section id="sc_commonProblems"> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/main/org/apache/zookeeper/server/TraceFormatter.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/TraceFormatter.java b/src/java/main/org/apache/zookeeper/server/TraceFormatter.java index 60d1cc7..66cc31c 100644 --- a/src/java/main/org/apache/zookeeper/server/TraceFormatter.java +++ b/src/java/main/org/apache/zookeeper/server/TraceFormatter.java @@ -29,7 +29,7 @@ import org.apache.zookeeper.ZooDefs.OpCode; public class TraceFormatter { - static String op2String(int op) { + public static String op2String(int op) { switch (op) { case OpCode.notification: return "notification"; http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java b/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java new file mode 100644 index 0000000..c4052e9 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +public class FilePadding { + private static final Logger LOG; + private static long preAllocSize = 65536 * 1024; + private static final ByteBuffer fill = ByteBuffer.allocateDirect(1); + + static { + LOG = LoggerFactory.getLogger(FileTxnLog.class); + + String size = System.getProperty("zookeeper.preAllocSize"); + if (size != null) { + try { + preAllocSize = Long.parseLong(size) * 1024; + } catch (NumberFormatException e) { + LOG.warn(size + " is not a valid value for preAllocSize"); + } + } + } + + private long currentSize; + + /** + * method to allow setting preallocate size + * of log file to pad the file. + * + * @param size the size to set to in bytes + */ + public static void setPreallocSize(long size) { + preAllocSize = size; + } + + public void setCurrentSize(long currentSize) { + this.currentSize = currentSize; + } + + /** + * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position + * + * @param fileChannel the fileChannel of the file to be padded + * @throws IOException + */ + long padFile(FileChannel fileChannel) throws IOException { + long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); + if (currentSize != newFileSize) { + fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); + currentSize = newFileSize; + } + return currentSize; + } + + /** + * Calculates a new file size with padding. We only return a new size if + * the current file position is sufficiently close (less than 4K) to end of + * file and preAllocSize is > 0. + * + * @param position the point in the file we have written to + * @param fileSize application keeps track of the current file size + * @param preAllocSize how many bytes to pad + * @return the new file size. It can be the same as fileSize if no + * padding was done. + * @throws IOException + */ + // VisibleForTesting + public static long calculateFileSizeWithPadding(long position, long fileSize, long preAllocSize) { + // If preAllocSize is positive and we are within 4KB of the known end of the file calculate a new file size + if (preAllocSize > 0 && position + 4096 >= fileSize) { + // If we have written more than we have previously preallocated we need to make sure the new + // file size is larger than what we already have + if (position > fileSize) { + fileSize = position + preAllocSize; + fileSize -= fileSize % preAllocSize; + } else { + fileSize += preAllocSize; + } + } + + return fileSize; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java index 3694c98..15c7211 100644 --- a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java +++ b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java @@ -91,9 +91,6 @@ import org.slf4j.LoggerFactory; public class FileTxnLog implements TxnLog { private static final Logger LOG; - static long preAllocSize = 65536 * 1024; - private static final ByteBuffer fill = ByteBuffer.allocateDirect(1); - public final static int TXNLOG_MAGIC = ByteBuffer.wrap("ZKLG".getBytes()).getInt(); @@ -107,14 +104,6 @@ public class FileTxnLog implements TxnLog { static { LOG = LoggerFactory.getLogger(FileTxnLog.class); - String size = System.getProperty("zookeeper.preAllocSize"); - if (size != null) { - try { - preAllocSize = Long.parseLong(size) * 1024; - } catch (NumberFormatException e) { - LOG.warn(size + " is not a valid value for preAllocSize"); - } - } /** Local variable to read fsync.warningthresholdms into */ Long fsyncWarningThreshold; if ((fsyncWarningThreshold = Long.getLong("zookeeper.fsync.warningthresholdms")) == null) @@ -132,8 +121,8 @@ public class FileTxnLog implements TxnLog { long dbId; private LinkedList<FileOutputStream> streamsToFlush = new LinkedList<FileOutputStream>(); - long currentSize; File logFileWrite = null; + private FilePadding filePadding = new FilePadding(); /** * constructor for FileTxnLog. Take the directory @@ -145,15 +134,6 @@ public class FileTxnLog implements TxnLog { } /** - * method to allow setting preallocate size - * of log file to pad the file. - * @param size the size to set to in bytes - */ - public static void setPreallocSize(long size) { - preAllocSize = size; - } - - /** * creates a checksum alogrithm to be used * @return the checksum used for this txnlog */ @@ -161,7 +141,6 @@ public class FileTxnLog implements TxnLog { return new Adler32(); } - /** * rollover the current log file to a new one. * @throws IOException @@ -213,18 +192,18 @@ public class FileTxnLog implements TxnLog { LOG.info("Creating new log file: " + Util.makeLogName(hdr.getZxid())); } - logFileWrite = new File(logDir, Util.makeLogName(hdr.getZxid())); - fos = new FileOutputStream(logFileWrite); - logStream=new BufferedOutputStream(fos); - oa = BinaryOutputArchive.getArchive(logStream); - FileHeader fhdr = new FileHeader(TXNLOG_MAGIC,VERSION, dbId); - fhdr.serialize(oa, "fileheader"); - // Make sure that the magic number is written before padding. - logStream.flush(); - currentSize = fos.getChannel().position(); - streamsToFlush.add(fos); - } - currentSize = padFile(fos.getChannel()); + logFileWrite = new File(logDir, Util.makeLogName(hdr.getZxid())); + fos = new FileOutputStream(logFileWrite); + logStream=new BufferedOutputStream(fos); + oa = BinaryOutputArchive.getArchive(logStream); + FileHeader fhdr = new FileHeader(TXNLOG_MAGIC,VERSION, dbId); + fhdr.serialize(oa, "fileheader"); + // Make sure that the magic number is written before padding. + logStream.flush(); + filePadding.setCurrentSize(fos.getChannel().position()); + streamsToFlush.add(fos); + } + filePadding.padFile(fos.getChannel()); byte[] buf = Util.marshallTxnEntry(hdr, txn); if (buf == null || buf.length == 0) { throw new IOException("Faulty serialization for header " + @@ -239,49 +218,6 @@ public class FileTxnLog implements TxnLog { } /** - * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position - * @param fileChannel the fileChannel of the file to be padded - * @throws IOException - */ - private long padFile(FileChannel fileChannel) throws IOException { - long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); - if (currentSize != newFileSize) { - fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); - currentSize = newFileSize; - } - return currentSize; - } - - /** - * Calculates a new file size with padding. We only return a new size if - * the current file position is sufficiently close (less than 4K) to end of - * file and preAllocSize is > 0. - * - * @param position the point in the file we have written to - * @param fileSize application keeps track of the current file size - * @param preAllocSize how many bytes to pad - * @return the new file size. It can be the same as fileSize if no - * padding was done. - * @throws IOException - */ - // VisibleForTesting - public static long calculateFileSizeWithPadding(long position, long fileSize, long preAllocSize) { - // If preAllocSize is positive and we are within 4KB of the known end of the file calculate a new file size - if (preAllocSize > 0 && position + 4096 >= fileSize) { - // If we have written more than we have previously preallocated we need to make sure the new - // file size is larger than what we already have - if (position > fileSize){ - fileSize = position + preAllocSize; - fileSize -= fileSize % preAllocSize; - } else { - fileSize += preAllocSize; - } - } - - return fileSize; - } - - /** * Find the log file that starts at, or just before, the snapshot. Return * this and all subsequent logs. Results are ordered by zxid of file, * ascending order. http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java new file mode 100644 index 0000000..432cc88 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java @@ -0,0 +1,280 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.apache.jute.BinaryInputArchive; +import org.apache.jute.BinaryOutputArchive; +import org.apache.jute.Record; +import org.apache.zookeeper.server.TraceFormatter; +import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.txn.TxnHeader; + +import java.io.Closeable; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.text.DateFormat; +import java.util.Date; +import java.util.Scanner; +import java.util.zip.Adler32; +import java.util.zip.Checksum; + +import static org.apache.zookeeper.server.persistence.FileTxnLog.TXNLOG_MAGIC; +import static org.apache.zookeeper.server.persistence.TxnLogToolkitCliParser.printHelpAndExit; + +public class TxnLogToolkit implements Closeable { + + static class TxnLogToolkitException extends Exception { + private static final long serialVersionUID = 1L; + private int exitCode; + + TxnLogToolkitException(int exitCode, String message, Object... params) { + super(String.format(message, params)); + this.exitCode = exitCode; + } + + int getExitCode() { + return exitCode; + } + } + + static class TxnLogToolkitParseException extends TxnLogToolkitException { + private static final long serialVersionUID = 1L; + + TxnLogToolkitParseException(int exitCode, String message, Object... params) { + super(exitCode, message, params); + } + } + + private File txnLogFile; + private boolean recoveryMode = false; + private boolean verbose = false; + private FileInputStream txnFis; + private BinaryInputArchive logStream; + + // Recovery mode + private int crcFixed = 0; + private FileOutputStream recoveryFos; + private BinaryOutputArchive recoveryOa; + private File recoveryLogFile; + private FilePadding filePadding = new FilePadding(); + private boolean force = false; + + /** + * @param args Command line arguments + */ + public static void main(String[] args) throws Exception { + final TxnLogToolkit lt = parseCommandLine(args); + try { + lt.dump(new InputStreamReader(System.in)); + lt.printStat(); + } catch (TxnLogToolkitParseException e) { + System.err.println(e.getMessage() + "\n"); + printHelpAndExit(e.getExitCode()); + } catch (TxnLogToolkitException e) { + System.err.println(e.getMessage()); + System.exit(e.getExitCode()); + } finally { + lt.close(); + } + } + + public TxnLogToolkit(boolean recoveryMode, boolean verbose, String txnLogFileName, boolean force) + throws FileNotFoundException, TxnLogToolkitException { + this.recoveryMode = recoveryMode; + this.verbose = verbose; + this.force = force; + txnLogFile = new File(txnLogFileName); + if (!txnLogFile.exists() || !txnLogFile.canRead()) { + throw new TxnLogToolkitException(1, "File doesn't exist or not readable: %s", txnLogFile); + } + if (recoveryMode) { + recoveryLogFile = new File(txnLogFile.toString() + ".fixed"); + if (recoveryLogFile.exists()) { + throw new TxnLogToolkitException(1, "Recovery file %s already exists or not writable", recoveryLogFile); + } + } + + openTxnLogFile(); + if (recoveryMode) { + openRecoveryFile(); + } + } + + public void dump(Reader input) throws Exception { + crcFixed = 0; + + FileHeader fhdr = new FileHeader(); + fhdr.deserialize(logStream, "fileheader"); + if (fhdr.getMagic() != TXNLOG_MAGIC) { + throw new TxnLogToolkitException(2, "Invalid magic number for %s", txnLogFile.getName()); + } + System.out.println("ZooKeeper Transactional Log File with dbid " + + fhdr.getDbid() + " txnlog format version " + + fhdr.getVersion()); + + if (recoveryMode) { + fhdr.serialize(recoveryOa, "fileheader"); + recoveryFos.flush(); + filePadding.setCurrentSize(recoveryFos.getChannel().position()); + } + + int count = 0; + while (true) { + long crcValue; + byte[] bytes; + try { + crcValue = logStream.readLong("crcvalue"); + bytes = logStream.readBuffer("txnEntry"); + } catch (EOFException e) { + System.out.println("EOF reached after " + count + " txns."); + return; + } + if (bytes.length == 0) { + // Since we preallocate, we define EOF to be an + // empty transaction + System.out.println("EOF reached after " + count + " txns."); + return; + } + Checksum crc = new Adler32(); + crc.update(bytes, 0, bytes.length); + if (crcValue != crc.getValue()) { + if (recoveryMode) { + if (!force) { + printTxn(bytes, "CRC ERROR"); + if (askForFix(input)) { + crcValue = crc.getValue(); + ++crcFixed; + } + } else { + crcValue = crc.getValue(); + printTxn(bytes, "CRC FIXED"); + ++crcFixed; + } + } else { + printTxn(bytes, "CRC ERROR"); + } + } + if (!recoveryMode || verbose) { + printTxn(bytes); + } + if (logStream.readByte("EOR") != 'B') { + throw new TxnLogToolkitException(1, "Last transaction was partial."); + } + if (recoveryMode) { + filePadding.padFile(recoveryFos.getChannel()); + recoveryOa.writeLong(crcValue, "crcvalue"); + recoveryOa.writeBuffer(bytes, "txnEntry"); + recoveryOa.writeByte((byte)'B', "EOR"); + } + count++; + } + } + + private boolean askForFix(Reader input) throws TxnLogToolkitException { + Scanner scanner = new Scanner(input); + try { + while (true) { + System.out.print("Would you like to fix it (Yes/No/Abort) ? "); + char answer = Character.toUpperCase(scanner.next().charAt(0)); + switch (answer) { + case 'Y': + return true; + case 'N': + return false; + case 'A': + throw new TxnLogToolkitException(0, "Recovery aborted."); + } + } + } finally { + scanner.close(); + } + } + + private void printTxn(byte[] bytes) throws IOException { + printTxn(bytes, ""); + } + + private void printTxn(byte[] bytes, String prefix) throws IOException { + TxnHeader hdr = new TxnHeader(); + Record txn = SerializeUtils.deserializeTxn(bytes, hdr); + String txns = String.format("%s session 0x%s cxid 0x%s zxid 0x%s %s %s", + DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.LONG).format(new Date(hdr.getTime())), + Long.toHexString(hdr.getClientId()), + Long.toHexString(hdr.getCxid()), + Long.toHexString(hdr.getZxid()), + TraceFormatter.op2String(hdr.getType()), + txn); + if (prefix != null && !"".equals(prefix.trim())) { + System.out.print(prefix + " - "); + } + if (txns.endsWith("\n")) { + System.out.print(txns); + } else { + System.out.println(txns); + } + } + + private void openTxnLogFile() throws FileNotFoundException { + txnFis = new FileInputStream(txnLogFile); + logStream = BinaryInputArchive.getArchive(txnFis); + } + + private void closeTxnLogFile() throws IOException { + if (txnFis != null) { + txnFis.close(); + } + } + + private void openRecoveryFile() throws FileNotFoundException { + recoveryFos = new FileOutputStream(recoveryLogFile); + recoveryOa = BinaryOutputArchive.getArchive(recoveryFos); + } + + private void closeRecoveryFile() throws IOException { + if (recoveryFos != null) { + recoveryFos.close(); + } + } + + private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolkitException, FileNotFoundException { + TxnLogToolkitCliParser parser = new TxnLogToolkitCliParser(); + parser.parse(args); + return new TxnLogToolkit(parser.isRecoveryMode(), parser.isVerbose(), parser.getTxnLogFileName(), parser.isForce()); + } + + private void printStat() { + if (recoveryMode) { + System.out.printf("Recovery file %s has been written with %d fixed CRC error(s)%n", recoveryLogFile, crcFixed); + } + } + + @Override + public void close() throws IOException { + if (recoveryMode) { + closeRecoveryFile(); + } + closeTxnLogFile(); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParser.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParser.java b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParser.java new file mode 100644 index 0000000..094500a --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParser.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +class TxnLogToolkitCliParser { + private String txnLogFileName; + private boolean recoveryMode; + private boolean verbose; + private boolean force; + + String getTxnLogFileName() { + return txnLogFileName; + } + + boolean isRecoveryMode() { + return recoveryMode; + } + + boolean isVerbose() { + return verbose; + } + + boolean isForce() { + return force; + } + + void parse(String[] args) throws TxnLogToolkit.TxnLogToolkitParseException { + if (args == null) { + throw new TxnLogToolkit.TxnLogToolkitParseException(1, "No arguments given"); + } + txnLogFileName = null; + for (String arg : args) { + if (arg.startsWith("--")) { + String par = arg.substring(2); + if ("help".equalsIgnoreCase(par)) { + printHelpAndExit(0); + } else if ("recover".equalsIgnoreCase(par)) { + recoveryMode = true; + } else if ("verbose".equalsIgnoreCase(par)) { + verbose = true; + } else if ("dump".equalsIgnoreCase(par)) { + recoveryMode = false; + } else if ("yes".equalsIgnoreCase(par)) { + force = true; + } else { + throw new TxnLogToolkit.TxnLogToolkitParseException(1, "Invalid argument: %s", par); + } + } else if (arg.startsWith("-")) { + String par = arg.substring(1); + if ("h".equalsIgnoreCase(par)) { + printHelpAndExit(0); + } else if ("r".equalsIgnoreCase(par)) { + recoveryMode = true; + } else if ("v".equalsIgnoreCase(par)) { + verbose = true; + } else if ("d".equalsIgnoreCase(par)) { + recoveryMode = false; + } else if ("y".equalsIgnoreCase(par)) { + force = true; + } else { + throw new TxnLogToolkit.TxnLogToolkitParseException(1, "Invalid argument: %s", par); + } + } else { + if (txnLogFileName != null) { + throw new TxnLogToolkit.TxnLogToolkitParseException(1, "Invalid arguments: more than one TXN log file given"); + } + txnLogFileName = arg; + } + } + + if (txnLogFileName == null) { + throw new TxnLogToolkit.TxnLogToolkitParseException(1, "Invalid arguments: TXN log file name missing"); + } + } + + static void printHelpAndExit(int exitCode) { + System.out.println("usage: TxnLogToolkit [-dhrvy] txn_log_file_name\n"); + System.out.println(" -d,--dump Dump mode. Dump all entries of the log file. (this is the default)"); + System.out.println(" -h,--help Print help message"); + System.out.println(" -r,--recover Recovery mode. Re-calculate CRC for broken entries."); + System.out.println(" -v,--verbose Be verbose in recovery mode: print all entries, not just fixed ones."); + System.out.println(" -y,--yes Non-interactive mode: repair all CRC errors without asking"); + System.exit(exitCode); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/test/data/invalidsnap/version-2/log.42 ---------------------------------------------------------------------- diff --git a/src/java/test/data/invalidsnap/version-2/log.42 b/src/java/test/data/invalidsnap/version-2/log.42 new file mode 100644 index 0000000..5385be5 Binary files /dev/null and b/src/java/test/data/invalidsnap/version-2/log.42 differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java index 5f54d0e..97cbc37 100644 --- a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java +++ b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java @@ -39,27 +39,27 @@ public class FileTxnLogTest extends ZKTestCase { @Test public void testInvalidPreallocSize() { Assert.assertEquals("file should not be padded", - 10 * KB, FileTxnLog.calculateFileSizeWithPadding(7 * KB, 10 * KB, 0)); + 10 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, 0)); Assert.assertEquals("file should not be padded", - 10 * KB, FileTxnLog.calculateFileSizeWithPadding(7 * KB, 10 * KB, -1)); + 10 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, -1)); } @Test public void testCalculateFileSizeWithPaddingWhenNotToCurrentSize() { Assert.assertEquals("file should not be padded", - 10 * KB, FileTxnLog.calculateFileSizeWithPadding(5 * KB, 10 * KB, 10 * KB)); + 10 * KB, FilePadding.calculateFileSizeWithPadding(5 * KB, 10 * KB, 10 * KB)); } @Test public void testCalculateFileSizeWithPaddingWhenCloseToCurrentSize() { Assert.assertEquals("file should be padded an additional 10 KB", - 20 * KB, FileTxnLog.calculateFileSizeWithPadding(7 * KB, 10 * KB, 10 * KB)); + 20 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, 10 * KB)); } @Test public void testFileSizeGreaterThanPosition() { Assert.assertEquals("file should be padded to 40 KB", - 40 * KB, FileTxnLog.calculateFileSizeWithPadding(31 * KB, 10 * KB, 10 * KB)); + 40 * KB, FilePadding.calculateFileSizeWithPadding(31 * KB, 10 * KB, 10 * KB)); } @Test @@ -69,7 +69,7 @@ public class FileTxnLogTest extends ZKTestCase { // Set a small preAllocSize (.5 MB) final int preAllocSize = 500 * KB; - fileTxnLog.setPreallocSize(preAllocSize); + FilePadding.setPreallocSize(preAllocSize); // Create dummy txn larger than preAllocSize // Since the file padding inserts a 0, we will fill the data with 0xff to ensure we corrupt the data if we put the 0 in the data http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParserTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParserTest.java b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParserTest.java new file mode 100644 index 0000000..ee4dc06 --- /dev/null +++ b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitCliParserTest.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class TxnLogToolkitCliParserTest { + + private TxnLogToolkitCliParser parser; + + @Before + public void setUp() { + parser = new TxnLogToolkitCliParser(); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitParseException.class) + public void testParseWithNoArguments() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(null); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitParseException.class) + public void testParseWithEmptyArgs() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[0]); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitParseException.class) + public void testParseWith2Filenames() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "file1.log", "file2.log "}); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitParseException.class) + public void testParseWithInvalidShortSwitch() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-v", "-i", "txnlog.txt" }); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitParseException.class) + public void testParseWithInvalidLongSwitch() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-v", "--invalid", "txnlog.txt" }); + } + + @Test + public void testParseRecoveryModeSwitchShort() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-r", "txnlog.txt"}); + assertThat("Recovery short switch should turn on recovery mode", parser.isRecoveryMode(), is(true)); + } + + @Test + public void testParseRecoveryModeSwitchLong() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "--recover", "txnlog.txt"}); + assertThat("Recovery long switch should turn on recovery mode", parser.isRecoveryMode(), is(true)); + } + + @Test + public void testParseVerboseModeSwitchShort() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-v", "txnlog.txt"}); + assertThat("Verbose short switch should turn on verbose mode", parser.isVerbose(), is(true)); + } + + @Test + public void testParseVerboseModeSwitchLong() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "--verbose", "txnlog.txt"}); + assertThat("Verbose long switch should turn on verbose mode", parser.isVerbose(), is(true)); + } + + @Test + public void testParseDumpModeSwitchShort() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-r", "txnlog.txt"}); // turn on + parser.parse(new String[] { "-d", "txnlog.txt"}); // turn off + assertThat("Dump short switch should turn off recover mode", parser.isRecoveryMode(), is(false)); + } + + @Test + public void testParseDumpModeSwitchLong() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-r", "txnlog.txt"}); // turn on + parser.parse(new String[] { "--dump", "txnlog.txt"}); // turn off + assertThat("Dump long switch should turn off recovery mode", parser.isRecoveryMode(), is(false)); + } + + @Test + public void testParseForceModeSwitchShort() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "-y", "txnlog.txt"}); + assertThat("Force short switch should turn on force mode", parser.isForce(), is(true)); + } + + @Test + public void testParseForceModeSwitchLong() throws TxnLogToolkit.TxnLogToolkitParseException { + parser.parse(new String[] { "--yes", "txnlog.txt"}); + assertThat("Force long switch should turn on force mode", parser.isForce(), is(true)); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java new file mode 100644 index 0000000..62557d1 --- /dev/null +++ b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.persistence; + +import org.apache.commons.io.FileUtils; +import org.apache.zookeeper.test.ClientBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.io.StringReader; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.matchers.JUnitMatchers.containsString; + + +public class TxnLogToolkitTest { + private static final File testData = new File( + System.getProperty("test.data.dir", "build/test/data")); + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private File mySnapDir; + + @Before + public void setUp() throws IOException { + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + File snapDir = new File(testData, "invalidsnap"); + mySnapDir = ClientBase.createTmpDir(); + FileUtils.copyDirectory(snapDir, mySnapDir); + } + + @After + public void tearDown() throws IOException { + System.setOut(System.out); + System.setErr(System.err); + mySnapDir.setWritable(true); + FileUtils.deleteDirectory(mySnapDir); + } + + @Test + public void testDumpMode() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + // no exception thrown + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitException.class) + public void testInitMissingFile() throws FileNotFoundException, TxnLogToolkit.TxnLogToolkitException { + // Arrange & Act + File logfile = new File("this_file_should_not_exists"); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); + } + + @Test(expected = TxnLogToolkit.TxnLogToolkitException.class) + public void testInitWithRecoveryFileExists() throws IOException, TxnLogToolkit.TxnLogToolkitException { + // Arrange & Act + File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); + File recoveryFile = new File(new File(mySnapDir, "version-2"), "log.274.fixed"); + recoveryFile.createNewFile(); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); + } + + @Test + public void testDumpWithCrcError() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + String output = outContent.toString(); + Pattern p = Pattern.compile("^CRC ERROR.*session 0x8061fac5ddeb0000 cxid 0x0 zxid 0x8800000002 createSession 30000$", Pattern.MULTILINE); + Matcher m = p.matcher(output); + assertTrue("Output doesn't indicate CRC error for the broken session id: " + output, m.find()); + } + + @Test + public void testRecoveryFixBrokenFile() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + String output = outContent.toString(); + assertThat(output, containsString("CRC FIXED")); + + // Should be able to dump the recovered logfile with no CRC error + outContent.reset(); + logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); + lt = new TxnLogToolkit(false, false, logfile.toString(), true); + lt.dump(null); + output = outContent.toString(); + assertThat(output, not(containsString("CRC ERROR"))); + } + + @Test + public void testRecoveryInteractiveMode() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), false); + + // Act + lt.dump(new StringReader("y\n")); + + // Assert + String output = outContent.toString(); + assertThat(output, containsString("CRC ERROR")); + + // Should be able to dump the recovered logfile with no CRC error + outContent.reset(); + logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); + lt = new TxnLogToolkit(false, false, logfile.toString(), true); + lt.dump(null); + output = outContent.toString(); + assertThat(output, not(containsString("CRC ERROR"))); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/126fb0f2/src/java/test/org/apache/zookeeper/test/ClientBase.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/test/ClientBase.java b/src/java/test/org/apache/zookeeper/test/ClientBase.java index 74d0eed..7de4f3c 100644 --- a/src/java/test/org/apache/zookeeper/test/ClientBase.java +++ b/src/java/test/org/apache/zookeeper/test/ClientBase.java @@ -56,6 +56,7 @@ import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ServerCnxnFactoryAccessor; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FilePadding; import org.apache.zookeeper.server.persistence.FileTxnLog; import org.apache.zookeeper.server.quorum.QuorumPeer; import org.apache.zookeeper.server.util.OSMXBean; @@ -467,7 +468,7 @@ public abstract class ClientBase extends ZKTestCase { // resulting in test Assert.failure (client timeout on first session). // set env and directly in order to handle static init/gc issues System.setProperty("zookeeper.preAllocSize", "100"); - FileTxnLog.setPreallocSize(100 * 1024); + FilePadding.setPreallocSize(100 * 1024); } protected void setUpAll() throws Exception {
