This is an automated email from the ASF dual-hosted git repository.
andor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 73cdd09 ZOOKEEPER-3179: Add snapshot compression to reduce the disk IO
73cdd09 is described below
commit 73cdd09051df33189351f7c42131724e3270eeac
Author: Yisong Yue <[email protected]>
AuthorDate: Wed Apr 10 13:19:02 2019 +0200
ZOOKEEPER-3179: Add snapshot compression to reduce the disk IO
Author: Yisong Yue <[email protected]>
Reviewers: [email protected]
Closes #690 from yisong-yue/ZOOKEEPER-3179
---
build.xml | 162 +++++-----
ivy.xml | 5 +-
zookeeper-server/pom.xml | 6 +
.../apache/zookeeper/server/SnapshotFormatter.java | 7 +-
.../zookeeper/server/persistence/FileSnap.java | 37 +--
.../zookeeper/server/persistence/SnapStream.java | 326 +++++++++++++++++++++
.../apache/zookeeper/server/persistence/Util.java | 89 ++----
.../zookeeper/server/ZooKeeperServerTest.java | 3 +-
.../server/persistence/SnapStreamTest.java | 141 +++++++++
9 files changed, 594 insertions(+), 182 deletions(-)
diff --git a/build.xml b/build.xml
index 38e5e0e..c0dc8fe 100644
--- a/build.xml
+++ b/build.xml
@@ -17,7 +17,7 @@
limitations under the License.
-->
-<project name="ZooKeeper" default="jar"
+<project name="ZooKeeper" default="jar"
xmlns:ivy="antlib:org.apache.ivy.ant"
xmlns:artifact="antlib:org.apache.maven.artifact.ant"
xmlns:maven="antlib:org.apache.maven.artifact.ant"
@@ -62,6 +62,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="kerby.version" value="1.1.0"/>
<property name="clover.version" value="4.2.1" />
+
+ <property name="snappy.version" value="1.1.7"/>
<property name="json.version" value="1.1.1"/>
<!-- ====================================================== -->
@@ -74,7 +76,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="name" value="zookeeper" />
<property environment="env"/>
-
+
<property name="version-major" value="3" />
<property name="version-minor" value="6" />
<property name="version-patch" value="0" />
@@ -86,7 +88,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="revision.dir" value="${basedir}/.revision" />
<property name="revision.properties" value="revision.properties" />
<property file="${basedir}/.revision/${revision.properties}" />
-
+
<property name="javac.target" value="1.8" />
<property name="javac.source" value="1.8" />
<property name="build.encoding" value="utf8" />
@@ -207,7 +209,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="jdiff.build.dir" value="${build.docs}/jdiff"/>
<property name="jdiff.xml.dir" value="${lib.dir}/jdiff"/>
<property name="jdiff.stable" value="3.1.1"/>
- <property name="jdiff.stable.javadoc"
+ <property name="jdiff.stable.javadoc"
value="http://hadoop.apache.org/zookeeper/docs/r${jdiff.stable}/api/"/>
<!-- eclipse property set -->
@@ -223,14 +225,14 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="staging_repo_id" value="apache.staging.https"/>
<property name="wagon-http.version" value="2.4"/>
<property name="snapshots_repo_id" value="apache.snapshots.https"/>
- <property name="asfrepo" value="https://repository.apache.org"/>
- <property name="snapshots_repo_url"
- value="${asfrepo}/content/repositories/snapshots"/>
+ <property name="asfrepo" value="https://repository.apache.org"/>
+ <property name="snapshots_repo_url"
+ value="${asfrepo}/content/repositories/snapshots"/>
<property name="staging_repo_url"
- value="${asfrepo}/service/local/staging/deploy/maven2"/>
- <property name="gpg-plugin"
+ value="${asfrepo}/service/local/staging/deploy/maven2"/>
+ <property name="gpg-plugin"
value="org.apache.maven.plugins:maven-gpg-plugin:1.4:sign-and-deploy-file"/>
- <property name="deploy-plugin"
+ <property name="deploy-plugin"
value="org.apache.maven.plugins:maven-deploy-plugin:2.8.1:deploy-file"/>
<property name="main-jar" value="${dist.maven.dir}/${final.name}.jar"/>
<property name="tests-jar"
value="${dist.maven.dir}/${final.name}-tests.jar"/>
@@ -294,7 +296,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<!-- ====================================================== -->
<!-- Generate and compile the Java files -->
<!-- ====================================================== -->
- <target name="init">
+ <target name="init">
<mkdir dir="${build.classes}" />
<mkdir dir="${ivy.lib}"/>
@@ -309,8 +311,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<format property="year" pattern="yyyy" timezone="GMT"/>
</tstamp>
</target>
-
-
+
+
<target name="generate_jute_parser"
depends="init,ivy-retrieve,ivy-retrieve-javacc">
<property name="jute_javacc.dir" value="${build.dir}/jute_compiler" />
<property name="jute_javacc.packagedir"
value="/org/apache/jute/compiler/generated" />
@@ -324,7 +326,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
javacchome="${ivy.javacc.lib}"
/>
</target>
-
+
<target name="jute" depends="generate_jute_parser">
<javac srcdir="${jute_javacc.dir}" destdir="${build.classes}"
includeantruntime="false"
target="${javac.target}" source="${javac.source}"
@@ -352,7 +354,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<pathelement path="${jute_javacc.dir}" />
</classpath>
</java>
-
+
<java classname="org.apache.jute.compiler.generated.Rcc" fork="true"
dir="${csrc_generated.dir}">
<arg value="-l" />
<arg value="c" />
@@ -372,13 +374,13 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<src path="${java.server.src.dir}" />
</javac>
</target>
-
+
<target name="git-revision" unless="lastRevision">
<mkdir dir="${revision.dir}" />
<condition property="shell.name" value="cmd" else="sh">
<os family="windows"/>
</condition>
- <condition property="revision.cmd.line"
+ <condition property="revision.cmd.line"
value="/c ${java.server.resources.dir}\lastRevision.bat"
else="${java.server.resources.dir}/lastRevision.sh">
<os family="windows"/>
</condition>
@@ -387,10 +389,10 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</exec>
<property file="${revision.dir}/${revision.properties}" />
</target>
-
+
<target name="version-info" depends="ver-gen,git-revision">
<mkdir dir="${src_generated.dir}" />
- <java classname="org.apache.zookeeper.version.util.VerGen" fork="true"
+ <java classname="org.apache.zookeeper.version.util.VerGen" fork="true"
dir="${src_generated.dir}">
<arg value="${version}" />
<arg value="${lastRevision}" />
@@ -400,12 +402,12 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</classpath>
</java>
</target>
-
+
<target name="build-generated"
depends="compile_jute,version-info,process-template,ivy-retrieve" >
<javac srcdir="${src_generated.dir}" destdir="${build.classes}"
includeantruntime="false"
target="${javac.target}" source="${javac.source}" debug="on"
encoding="${build.encoding}"
classpath="${ivy.lib}/audience-annotations-${audience-annotations.version}.jar"
/>
</target>
-
+
<target name="ivy-download" unless="ivy.jar.exists" depends="init">
<delete dir="${lib.dir}"
includes="ivy-*.jar" excludes="ivy-${ivy.version}.jar"/>
@@ -417,7 +419,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<taskdef resource="org/apache/ivy/ant/antlib.xml"
uri="antlib:org.apache.ivy.ant" classpathref="java.classpath"/>
<!-- ensure that ivy taskdef is only run once, otw ant will error -->
- <property name="ivy.initialized" value="true"/>
+ <property name="ivy.initialized" value="true"/>
</target>
<target name="ivy-init" depends="ivy-download,ivy-taskdef">
@@ -491,7 +493,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</filterchain>
</copy>
</target>
-
+
<target name="ivy-retrieve-mvn-ant-task" depends="init,ivy-init">
<ivy:retrieve settingsRef="${ant.project.name}" conf="mvn-ant-task"
pattern="${ivy.lib}/[artifact]-[revision].[ext]"/>
@@ -580,7 +582,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<link href="${javadoc.link.java}"/>
<classpath refid="java.classpath"/>
</javadoc>
- </target>
+ </target>
<target name="javadoc" depends="jar" description="Generate javadoc">
<mkdir dir="${build.javadoc}"/>
@@ -624,17 +626,17 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<classpath>
<fileset dir="${basedir}">
<include name="${jar.name}"/>
- </fileset>
+ </fileset>
<path refid="java.classpath"/>
</classpath>
</javadoc>
- </target>
+ </target>
<!-- ====================================================== -->
<!-- Make zookeeper.jar -->
<!-- ====================================================== -->
-
-
+
+
<target name="jar" depends="compile">
<java classname="org.apache.zookeeper.Version" fork="true"
outputproperty="revision" errorproperty="revision.error"
failonerror="true">
@@ -645,16 +647,16 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</classpath>
</java>
<exec executable="hostname" outputproperty="host.name"/>
-
+
<!-- Ensure that OSGi versions order properly whatever suffix is used
by -->
<!-- using DEV and REL at the start of the qualifier. DEV versions also
-->
<!-- use a timestamp to ensure later snapshots have higher versions -->
- <condition property="version-osgi"
+ <condition property="version-osgi"
value="${version-base}.DEV-${DSTAMP}-${TSTAMP}-${version-suffix}"
else="${version-base}.REL-${version-suffix}">
<contains string="${version-suffix}" substring="SNAPSHOT" />
</condition>
-
+
<jar jarfile="${build.dir}/${final.name}.jar">
<fileset file="LICENSE.txt" />
<fileset dir="${build.classes}" excludes="**/.generated"/>
@@ -667,11 +669,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<attribute name="Built-At" value="${build.time}"/>
<attribute name="Built-On" value="${host.name}" />
<attribute name="Implementation-Title"
value="org.apache.zookeeper"/>
- <attribute name="Implementation-Version" value="${revision}"/>
+ <attribute name="Implementation-Version" value="${revision}"/>
<attribute name="Implementation-Vendor" value="The Apache
Software Foundation"/>
-
+
<!-- The following are OSGi manifest headers -->
- <!-- currently hardcoded, when things get more complicated we
could use BND
+ <!-- currently hardcoded, when things get more complicated we
could use BND
http://www.aqute.biz/Code/Bnd to generate them -->
<attribute name="Bundle-Vendor" value="The Apache Software
Foundation"/>
<attribute name="Bundle-Name" value="ZooKeeper Bundle"/>
@@ -685,7 +687,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</manifest>
</jar>
</target>
-
+
<!-- ====================================================== -->
<!-- Make zookeeper-bin.jar -->
<!-- ====================================================== -->
@@ -708,11 +710,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<attribute name="Built-At" value="${build.time}"/>
<attribute name="Built-On" value="${host.name}" />
<attribute name="Implementation-Title"
value="org.apache.zookeeper"/>
- <attribute name="Implementation-Version" value="${revision}"/>
+ <attribute name="Implementation-Version" value="${revision}"/>
<attribute name="Implementation-Vendor" value="The Apache
Software Foundation"/>
-
+
<!-- The following are OSGi manifest headers -->
- <!-- currently hardcoded, when things get more complicated we
could use BND
+ <!-- currently hardcoded, when things get more complicated we
could use BND
http://www.aqute.biz/Code/Bnd to generate them -->
<attribute name="Bundle-Vendor" value="The Apache Software
Foundation"/>
<attribute name="Bundle-Name" value="ZooKeeper Bundle"/>
@@ -741,7 +743,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<attribute name="Built-At" value="${build.time}"/>
<attribute name="Built-On" value="${host.name}" />
<attribute name="Implementation-Title"
value="org.apache.zookeeper"/>
- <attribute name="Implementation-Version" value="${revision}"/>
+ <attribute name="Implementation-Version" value="${revision}"/>
<attribute name="Implementation-Vendor" value="The Apache
Software Foundation"/>
</manifest>
</jar>
@@ -759,7 +761,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<attribute name="Built-At" value="${build.time}"/>
<attribute name="Built-On" value="${host.name}" />
<attribute name="Implementation-Title"
value="org.apache.zookeeper"/>
- <attribute name="Implementation-Version" value="${revision}"/>
+ <attribute name="Implementation-Version" value="${revision}"/>
<attribute name="Implementation-Vendor" value="The Apache
Software Foundation"/>
</manifest>
</jar>
@@ -777,7 +779,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<attribute name="Built-At" value="${build.time}"/>
<attribute name="Built-On" value="${host.name}" />
<attribute name="Implementation-Title"
value="org.apache.zookeeper"/>
- <attribute name="Implementation-Version" value="${revision}"/>
+ <attribute name="Implementation-Version" value="${revision}"/>
<attribute name="Implementation-Vendor" value="The Apache
Software Foundation"/>
</manifest>
</jar>
@@ -788,7 +790,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<!-- ================================================================== -->
<!-- -->
<!-- ================================================================== -->
- <target name="package"
+ <target name="package"
depends="jar,bin-jar,src-jar,javadoc-jar,test-jar,api-report,create-cppunit-configure,compile-test"
description="Build distribution">
<mkdir dir="${dist.dir}"/>
@@ -813,9 +815,9 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="package.share" value=""/>
<fileset file="${contrib.dir}/build.xml"/>
<fileset file="${recipes.dir}/build.xml"/>
- </subant>
+ </subant>
- <copy todir="${dist.dir}">
+ <copy todir="${dist.dir}">
<fileset file="${build.dir}/${final.name}.jar"/>
</copy>
@@ -825,8 +827,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<mkdir dir="${dist.maven.dir}"/>
<copy file="${build.dir}/${final.name}-bin.jar"
- tofile="${dist.maven.dir}/${final.name}.jar"/>
- <copy todir="${dist.maven.dir}">
+ tofile="${dist.maven.dir}/${final.name}.jar"/>
+ <copy todir="${dist.maven.dir}">
<fileset file="${build.dir}/${final.name}-sources.jar"/>
<fileset file="${build.dir}/${final.name}-javadoc.jar"/>
</copy>
@@ -848,7 +850,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<checksum file="${dist.maven.dir}/${final.name}.pom" algorithm="sha1"/>
<copy file="${build.dir}/${final.name}-test.jar"
- tofile="${dist.maven.dir}/${final.name}-tests.jar"/>
+ tofile="${dist.maven.dir}/${final.name}-tests.jar"/>
<checksum file="${dist.maven.dir}/${final.name}-tests.jar"
algorithm="sha1"/>
<checksum file="${dist.maven.dir}/${final.name}-tests.jar"
algorithm="md5"/>
@@ -895,7 +897,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<copy todir="${dist.dir}/zookeeper-server/src/main/resources">
<fileset file="${java.server.resources.dir}/pom.template"/>
</copy>
-
+
<chmod perm="ugo+x" type="file" parallel="false">
<fileset dir="${dist.dir}/bin"/>
<fileset dir="${dist.dir}/zookeeper-contrib/">
@@ -934,7 +936,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<fileset file="${recipes.dir}/build.xml"/>
</subant>
- <copy todir="${dist.dir}/share/zookeeper">
+ <copy todir="${dist.dir}/share/zookeeper">
<fileset file="${build.dir}/${final.name}.jar"/>
</copy>
@@ -944,12 +946,12 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<mkdir dir="${dist.maven.dir}"/>
<copy file="${build.dir}/${final.name}-bin.jar"
- tofile="${dist.maven.dir}/${final.name}.jar"/>
- <copy todir="${dist.maven.dir}">
+ tofile="${dist.maven.dir}/${final.name}.jar"/>
+ <copy todir="${dist.maven.dir}">
<fileset file="${build.dir}/${final.name}-sources.jar"/>
<fileset file="${build.dir}/${final.name}-javadoc.jar"/>
</copy>
-
+
<checksum file="${dist.maven.dir}/${final.name}.jar" algorithm="md5"/>
<checksum file="${dist.maven.dir}/${final.name}.jar" algorithm="sha1"/>
<checksum file="${dist.maven.dir}/${final.name}-sources.jar"
algorithm="md5"/>
@@ -965,8 +967,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</ivy:makepom>
<checksum file="${dist.maven.dir}/${name}.pom" algorithm="md5"/>
<checksum file="${dist.maven.dir}/${name}.pom" algorithm="sha1"/>
-
-
+
+
<copy file="${build.dir}/${final.name}-test.jar"
tofile="${dist.maven.dir}/${final.name}-tests.jar"/>
<checksum file="${dist.maven.dir}/${final.name}-tests.jar"
algorithm="sha1"/>
@@ -1153,7 +1155,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<target name="signanddeploy" if="staging">
<deploy repo-url="${staging_repo_url}" repo-id="${staging_repo_id}"
plugin="${gpg-plugin}" profile="-Pgpg"/>
- </target>
+ </target>
<target name="simpledeploy" unless="staging">
<deploy plugin="${deploy-plugin}"/>
@@ -1162,7 +1164,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<!-- ====================================================== -->
<!-- mvn-install. Installing the jar and pom file to .m2 -->
<!-- ====================================================== -->
-
+
<target name="mvn-taskdef" depends="ivy-retrieve-mvn-ant-task">
<typedef resource="org/apache/maven/artifact/ant/antlib.xml"
uri="antlib:org.apache.maven.artifact.ant"
classpathref="mvn-ant-task-classpath"/>
@@ -1174,11 +1176,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<echo message="${dist.maven.dir}/${final.name}-sources.jar" />
<echo message="${dist.maven.dir}/${final.name}-javadoc.jar" />
<echo message="${dist.maven.dir}/${final.name}-tests.jar" />
-
+
<artifact:pom id="zookeeper-pom" file="${dist.maven.dir}/${name}.pom"/>
<echo>The version is ${zookeeper-pom.version}</echo>
<echo message="${dist.maven.dir}/${final.name}.jar" />
-
+
<artifact:install file="${dist.maven.dir}/${final.name}.jar">
<pom refid="zookeeper-pom" />
<attach file="${dist.maven.dir}/${final.name}.jar" type="jar"/>
@@ -1199,7 +1201,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<delete dir="${csrc_generated.dir}" />
<delete file="${lib.dir}/Null.java"/>
<delete file="${lib.dir}/rats.jar" />
- <delete file="${jdiff.xml.dir}/${name}_${version}.xml"/>
+ <delete file="${jdiff.xml.dir}/${name}_${version}.xml"/>
<delete file="${jar.name}" />
<delete dir="${distribution}"/>
<delete dir="${revision.dir}"/>
@@ -1209,9 +1211,9 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</target>
<target name="clean-contrib">
- <subant target="clean">
+ <subant target="clean">
<fileset file="${contrib.dir}/build.xml"/>
- </subant>
+ </subant>
</target>
<target name="clean-recipes">
@@ -1376,7 +1378,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<condition property="need.cppunit.configure">
<not> <available file="${c.src.dir}/configure"/> </not>
</condition>
- </target>
+ </target>
<target name="verify-cppunit-makefile-gcov">
<fileset id="fileset.makefile.gcov.enabled" dir="${test.cppunit.dir}"
erroronmissingdir="false">
@@ -1395,11 +1397,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</condition>
<echo message="delete.cppunit.makefile = ${delete.cppunit.makefile}"/>
</target>
-
+
<target name="delete-cppunit-makefile" if="delete.cppunit.makefile">
<delete file="${test.cppunit.dir}/Makefile"/>
</target>
-
+
<target name="check-cppunit-makefile"
depends="init,verify-cppunit-makefile-gcov,delete-cppunit-makefile" >
<condition property="need.cppunit.makefile">
<not> <available file="${test.cppunit.dir}/Makefile"/>
</not>
@@ -1409,7 +1411,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<!--
1. If we have a Makefile it will handle up-to-date check and also
regenerate the configure script if missing. (done)
- 2. If we don't have a Makefile use the configure script to
+ 2. If we don't have a Makefile use the configure script to
regenerate it. (done)
3. If we don't have a Makefile nor a configure script then it's
last resort and run autoreconf, then configure (done)
@@ -1424,7 +1426,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</exec>
</target>
- <target name="create-cppunit-makefile" depends="check-cppunit-makefile"
+ <target name="create-cppunit-makefile" depends="check-cppunit-makefile"
if="need.cppunit.makefile">
<antcall target="create-cppunit-configure">
<param name="cppunit" value="true"/>
@@ -1447,7 +1449,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<antcall target="test-cppunit">
<param name="cppunit" value="true"/>
</antcall>
- </target>
+ </target>
<target name="test-cppunit"
@@ -1463,7 +1465,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<arg line="clean check"/>
</exec>
</target>
-
+
<target name="test-unit-category">
<property name="test.category" value="Unit"/>
</target>
@@ -1487,11 +1489,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<antcall target="test-core"/>
<antcall target="test-contrib"/>
</target>
-
+
<target name="test-contrib" description="to run contrib tests">
<!-- yet to implement -->
</target>
-
+
<target name="test-core-java" depends="test-init, test-category,
junit.run"/>
<target name="test-core-cppunit" depends="test-init, test-category,
call-test-cppunit"/>
@@ -1587,10 +1589,10 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<property name="findbugs.exclude.file"
value="${server.test.resource.dir}/findbugsExcludeFile.xml" />
<property name="findbugs.report.htmlfile"
value="${findbugs.out.dir}/zookeeper-findbugs-report.html" />
<property name="findbugs.report.xmlfile"
value="${findbugs.out.dir}/zookeeper-findbugs-report.xml" />
- <taskdef name="findbugs"
classname="edu.umd.cs.findbugs.anttask.FindBugsTask"
+ <taskdef name="findbugs"
classname="edu.umd.cs.findbugs.anttask.FindBugsTask"
classpath="${findbugs.home}/lib/findbugs-ant.jar" />
<mkdir dir="${findbugs.out.dir}" />
- <findbugs home="${findbugs.home}" output="xml:withMessages"
excludeFilter="${findbugs.exclude.file}"
+ <findbugs home="${findbugs.home}" output="xml:withMessages"
excludeFilter="${findbugs.exclude.file}"
outputFile="${findbugs.report.xmlfile}" effort="max"
jvmargs="-Xmx512M">
<auxClasspath>
<fileset dir="${ivy.lib}">
@@ -1600,7 +1602,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
<sourcePath path="${java.server.src.dir}" />
<class location="${build.dir}/${final.name}.jar" />
</findbugs>
- <xslt style="${findbugs.home}/src/xsl/default.xsl"
in="${findbugs.report.xmlfile}"
+ <xslt style="${findbugs.home}/src/xsl/default.xsl"
in="${findbugs.report.xmlfile}"
out="${findbugs.report.htmlfile}" />
</target>
@@ -1633,7 +1635,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</target>
<target name="cobertura-test" depends="test-init,cobertura-instrument">
- <junit showoutput="${test.output}" printsummary="yes"
haltonfailure="no" fork="yes"
+ <junit showoutput="${test.output}" printsummary="yes"
haltonfailure="no" fork="yes"
maxmemory="${test.junit.maxmem}" dir="${basedir}"
timeout="${test.timeout}"
errorProperty="tests.failed" failureProperty="tests.failed">
<sysproperty key="build.test.dir" value="${test.tmp.dir}" />
@@ -1805,12 +1807,12 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</target>
<target name="findbugs.check" depends="check-for-findbugs"
unless="findbugs.present">
- <fail message="'findbugs.home' is not defined. Please pass
-Dfindbugs.home=<base of Findbugs installation>
+ <fail message="'findbugs.home' is not defined. Please pass
-Dfindbugs.home=<base of Findbugs installation>
to Ant on the command-line." />
</target>
<target name="patch.check" unless="patch.file">
- <fail message="'patch.file' is not defined. Please pass
-Dpatch.file=<location of patch file>
+ <fail message="'patch.file' is not defined. Please pass
-Dpatch.file=<location of patch file>
to Ant on the command-line." />
</target>
@@ -1894,17 +1896,17 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
</classpath>
</javadoc>
</target>
-
+
<target name="write-null">
<exec executable="touch">
<arg value="${jdiff.home}/Null.java"/>
</exec>
- </target>
+ </target>
<target name="api-report" depends="api-xml">
<mkdir dir="${jdiff.build.dir}"/>
<javadoc destdir="${jdiff.build.dir}"
- excludepackagenames="org.apache.jute"
+ excludepackagenames="org.apache.jute"
sourceFiles="${jdiff.home}/Null.java">
<fileset dir="${java.server.src.dir}"/>
<doclet name="jdiff.JDiff"
@@ -1979,7 +1981,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
output="${build.dir.eclipse-test-classes}" />
<source path="${bench.src.dir}"
output="${build.dir.eclipse-test-classes}" />
-
+
<output path="${build.dir.eclipse-main-classes}" />
<library pathref="default.path.id" exported="true" />
<library pathref="junit.path.id" exported="false" />
diff --git a/ivy.xml b/ivy.xml
index 96f9cde..6125980 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -76,7 +76,7 @@
<exclude org="commons-cli" module="commons-cli"/>
</dependency>
<!-- force the tests to pull the latest commons-collections jar -->
- <dependency org="commons-collections" name="commons-collections"
+ <dependency org="commons-collections" name="commons-collections"
rev="${commons-collections.version}" conf="test->default"/>
<dependency org="org.bouncycastle" name="bcprov-jdk15on"
rev="${bouncycastle.version}" conf="test->default"/>
@@ -87,7 +87,7 @@
<dependency org="xerces" name="xerces" rev="${xerces.version}"
conf="jdiff->default"/>
- <dependency org="org.apache.rat" name="apache-rat-tasks"
+ <dependency org="org.apache.rat" name="apache-rat-tasks"
rev="${apache-rat-tasks.version}" conf="releaseaudit->default">
<exclude org="commons-collections" module="commons-collections"/>
</dependency>
@@ -141,6 +141,7 @@
conf="optional->default"/>
<dependency org="com.fasterxml.jackson.core" name="jackson-databind"
rev="${jackson.version}" conf="optional->default"/>
+ <dependency org="org.xerial.snappy" name="snappy-java"
rev="${snappy.version}"/>
<dependency org="org.openclover" name="clover" rev="${clover.version}"
conf="clover->default"/>
diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml
index 0c24014..b8ef050 100755
--- a/zookeeper-server/pom.xml
+++ b/zookeeper-server/pom.xml
@@ -142,6 +142,12 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>1.1.7</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotFormatter.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotFormatter.java
index 50230ed..b93a8de 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotFormatter.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotFormatter.java
@@ -19,8 +19,8 @@
package org.apache.zookeeper.server;
import java.io.BufferedInputStream;
-import java.io.FileInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Base64;
@@ -37,6 +37,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.ZKUtil;
import org.apache.zookeeper.data.StatPersisted;
import org.apache.zookeeper.server.persistence.FileSnap;
+import org.apache.zookeeper.server.persistence.SnapStream;
import org.apache.zookeeper.server.persistence.Util;
import org.json.simple.JSONValue;
@@ -97,9 +98,7 @@ public class SnapshotFormatter {
public void run(String snapshotFileName, boolean dumpData, boolean
dumpJson)
throws IOException {
File snapshotFile = new File(snapshotFileName);
- try (InputStream is = new CheckedInputStream(
- new BufferedInputStream(new FileInputStream(snapshotFileName)),
- new Adler32())) {
+ try (InputStream is = SnapStream.getInputStream(snapshotFile)) {
InputArchive ia = BinaryInputArchive.getArchive(is);
FileSnap fileSnap = new FileSnap(null);
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileSnap.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileSnap.java
index 5cdfadb..3b9d030 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileSnap.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileSnap.java
@@ -17,11 +17,7 @@
*/
package org.apache.zookeeper.server.persistence;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -29,7 +25,6 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.zip.Adler32;
import java.util.zip.CheckedInputStream;
import java.util.zip.CheckedOutputStream;
@@ -82,15 +77,10 @@ public class FileSnap implements SnapShot {
for (int i = 0, snapListSize = snapList.size(); i < snapListSize; i++)
{
snap = snapList.get(i);
LOG.info("Reading snapshot " + snap);
- try (InputStream snapIS = new BufferedInputStream(new
FileInputStream(snap));
- CheckedInputStream crcIn = new CheckedInputStream(snapIS, new
Adler32())) {
- InputArchive ia = BinaryInputArchive.getArchive(crcIn);
+ try (CheckedInputStream snapIS = SnapStream.getInputStream(snap)) {
+ InputArchive ia = BinaryInputArchive.getArchive(snapIS);
deserialize(dt, sessions, ia);
- long checkSum = crcIn.getChecksum().getValue();
- long val = ia.readLong("val");
- if (val != checkSum) {
- throw new IOException("CRC corruption in snapshot : " +
snap);
- }
+ SnapStream.checkSealIntegrity(snapIS, ia);
foundValid = true;
break;
} catch (IOException e) {
@@ -136,12 +126,12 @@ public class FileSnap implements SnapShot {
}
/**
- * find the last (maybe) valid n snapshots. this does some
+ * find the last (maybe) valid n snapshots. this does some
* minor checks on the validity of the snapshots. It just
* checks for / at the end of the snapshot. This does
* not mean that the snapshot is truly valid but is
- * valid with a high probability. also, the most recent
- * will be first on the list.
+ * valid with a high probability. also, the most recent
+ * will be first on the list.
* @param n the number of most recent snapshots
* @return the last n snapshots (the number might be
* less than n in case enough snapshots are not available).
@@ -156,7 +146,7 @@ public class FileSnap implements SnapShot {
// from the valid snapshot and continue
// until we find a valid one
try {
- if (Util.isValidSnapshot(f)) {
+ if (SnapStream.isValidSnapshot(f)) {
list.add(f);
count++;
if (count == n) {
@@ -221,18 +211,11 @@ public class FileSnap implements SnapShot {
public synchronized void serialize(DataTree dt, Map<Long, Integer>
sessions, File snapShot, boolean fsync)
throws IOException {
if (!close) {
- try (CheckedOutputStream crcOut =
- new CheckedOutputStream(new
BufferedOutputStream(fsync ? new AtomicFileOutputStream(snapShot) :
-
new FileOutputStream(snapShot)),
- new Adler32())) {
- //CheckedOutputStream cout = new CheckedOutputStream()
- OutputArchive oa = BinaryOutputArchive.getArchive(crcOut);
+ try (CheckedOutputStream snapOS =
SnapStream.getOutputStream(snapShot)) {
+ OutputArchive oa = BinaryOutputArchive.getArchive(snapOS);
FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId);
serialize(dt, sessions, oa, header);
- long val = crcOut.getChecksum().getValue();
- oa.writeLong(val, "val");
- oa.writeString("/", "path");
- crcOut.flush();
+ SnapStream.sealStream(snapOS, oa);
}
}
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/SnapStream.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/SnapStream.java
new file mode 100644
index 0000000..f0d18d4
--- /dev/null
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/SnapStream.java
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.persistence;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.zip.Adler32;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.jute.InputArchive;
+import org.apache.jute.OutputArchive;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xerial.snappy.SnappyCodec;
+import org.xerial.snappy.SnappyInputStream;
+import org.xerial.snappy.SnappyOutputStream;
+
+/**
+ * Represent the Stream used in serialize and deserialize the Snapshot.
+ */
+public class SnapStream {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapStream.class);
+
+ public static final String ZOOKEEPER_SHAPSHOT_STREAM_MODE =
+ "zookeeper.snapshot.compression.method";
+
+ private static StreamMode streamMode =
+ StreamMode.fromString(
+ System.getProperty(ZOOKEEPER_SHAPSHOT_STREAM_MODE,
+ StreamMode.DEFAULT_MODE.getName()));
+
+ static {
+ LOG.info(ZOOKEEPER_SHAPSHOT_STREAM_MODE + "=" + streamMode);
+ }
+
+ public static enum StreamMode {
+ GZIP("gz"),
+ SNAPPY("snappy"),
+ CHECKED("");
+
+ public static final StreamMode DEFAULT_MODE = CHECKED;
+
+ private String name;
+
+ StreamMode(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getFileExtension() {
+ return name.isEmpty() ? "" : "." + name;
+ }
+
+ public static StreamMode fromString(String name) {
+ for (StreamMode c : values()) {
+ if (c.getName().compareToIgnoreCase(name) == 0) {
+ return c;
+ }
+ }
+ return DEFAULT_MODE;
+ }
+ }
+
+ /**
+ * Return the CheckedInputStream based on the extension of the fileName.
+ *
+ * @param fileName the file the InputStream read from
+ * @return the specific InputStream
+ * @throws IOException
+ */
+ public static CheckedInputStream getInputStream(File file) throws
IOException {
+ FileInputStream fis = new FileInputStream(file);
+ InputStream is;
+ switch (getStreamMode(file.getName())) {
+ case GZIP:
+ is = new GZIPInputStream(fis);
+ break;
+ case SNAPPY:
+ is = new SnappyInputStream(fis);
+ break;
+ case CHECKED:
+ default:
+ is = new BufferedInputStream(fis);
+ }
+ return new CheckedInputStream(is, new Adler32());
+ }
+
+ /**
+ * Return the OutputStream based on predefined stream mode.
+ *
+ * @param fileName the file the OutputStream writes to
+ * @return the specific OutputStream
+ * @throws IOException
+ */
+ public static CheckedOutputStream getOutputStream(File file) throws
IOException {
+ FileOutputStream fos = new FileOutputStream(file);
+ OutputStream os;
+ switch (streamMode) {
+ case GZIP:
+ os = new GZIPOutputStream(fos);
+ break;
+ case SNAPPY:
+ os = new SnappyOutputStream(fos);
+ break;
+ case CHECKED:
+ default:
+ os = new BufferedOutputStream(fos);
+ }
+ return new CheckedOutputStream(os, new Adler32());
+ }
+
+ /**
+ * Write specific seal to the OutputArchive and close the OutputStream.
+ * Currently, only CheckedOutputStream will write it's checkSum to the
+ * end of the stream.
+ *
+ */
+ public static void sealStream(CheckedOutputStream os, OutputArchive oa)
+ throws IOException {
+ long val = os.getChecksum().getValue();
+ oa.writeLong(val, "val");
+ oa.writeString("/", "path");
+ }
+
+ /**
+ * Verify the integrity of the seal, only CheckedInputStream will verify
+ * the checkSum of the content.
+ *
+ */
+ static void checkSealIntegrity(CheckedInputStream is, InputArchive ia)
+ throws IOException {
+ long checkSum = is.getChecksum().getValue();
+ long val = ia.readLong("val");
+ if (val != checkSum) {
+ throw new IOException("CRC corruption");
+ }
+ }
+
+ /**
+ * Verifies that the file is a valid snapshot. Snapshot may be invalid if
+ * it's incomplete as in a situation when the server dies while in the
+ * process of storing a snapshot. Any files that are improperly formated
+ * or corrupted are invalid. Any file that is not a snapshot is also an
+ * invalid snapshot.
+ *
+ * @param file file to verify
+ * @return true if the snapshot is valid
+ * @throws IOException
+ */
+ public static boolean isValidSnapshot(File file) throws IOException {
+ if (file == null || Util.getZxidFromName(file.getName(),
FileSnap.SNAPSHOT_FILE_PREFIX) == -1) {
+ return false;
+ }
+
+ String fileName = file.getName();
+ if (Util.getZxidFromName(fileName, "snapshot") == -1) {
+ return false;
+ }
+
+ boolean isValid = false;
+ switch (getStreamMode(fileName)) {
+ case GZIP:
+ isValid = isValidGZipStream(file);
+ break;
+ case SNAPPY:
+ isValid = isValidSnappyStream(file);
+ break;
+ case CHECKED:
+ default:
+ isValid = isValidCheckedStream(file);
+ }
+ return isValid;
+ }
+
+ public static void setStreamMode(StreamMode mode) {
+ streamMode = mode;
+ }
+
+ public static StreamMode getStreamMode() {
+ return streamMode;
+ }
+
+ /**
+ * Detect the stream mode from file name extension
+ *
+ * @param fileName
+ * @return
+ */
+ public static StreamMode getStreamMode(String fileName) {
+ String[] splitSnapName = fileName.split("\\.");
+
+ // Use file extension to detect format
+ if (splitSnapName.length > 1) {
+ String mode = splitSnapName[splitSnapName.length - 1];
+ return StreamMode.fromString(mode);
+ }
+
+ return StreamMode.CHECKED;
+ }
+
+ /**
+ * Certify the GZip stream integrity by checking the header
+ * for the GZip magic string
+ *
+ * @param f file to verify
+ * @return true if it has the correct GZip magic string
+ * @throws IOException
+ */
+ private static boolean isValidGZipStream(File f) throws IOException {
+ byte[] byteArray = new byte[2];
+ try (FileInputStream fis = new FileInputStream(f)) {
+ if (2 != fis.read(byteArray, 0, 2)) {
+ LOG.error("Read incorrect number of bytes from " +
f.getName());
+ return false;
+ }
+ ByteBuffer bb = ByteBuffer.wrap(byteArray);
+ byte[] magicHeader = new byte[2];
+ bb.get(magicHeader, 0, 2);
+ int magic = magicHeader[0] & 0xff | ((magicHeader[1] << 8) &
0xff00);
+ return magic == GZIPInputStream.GZIP_MAGIC;
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to open file " + f.getName() + " : ", e);
+ return false;
+ }
+ }
+
+ /**
+ * Certify the Snappy stream integrity by checking the header
+ * for the Snappy magic string
+ *
+ * @param f file to verify
+ * @return true if it has the correct Snappy magic string
+ * @throws IOException
+ */
+ private static boolean isValidSnappyStream(File f) throws IOException {
+ byte[] byteArray = new byte[SnappyCodec.MAGIC_LEN];
+ try (FileInputStream fis = new FileInputStream(f)) {
+ if (SnappyCodec.MAGIC_LEN != fis.read(byteArray, 0,
SnappyCodec.MAGIC_LEN)) {
+ LOG.error("Read incorrect number of bytes from " +
f.getName());
+ return false;
+ }
+ ByteBuffer bb = ByteBuffer.wrap(byteArray);
+ byte[] magicHeader = new byte[SnappyCodec.MAGIC_LEN];
+ bb.get(magicHeader, 0, SnappyCodec.MAGIC_LEN);
+ return Arrays.equals(magicHeader, SnappyCodec.getMagicHeader());
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to open file " + f.getName() + " : ", e);
+ return false;
+ }
+ }
+
+ /**
+ * Certify the Checked stream integrity by checking the header
+ * length and format
+ *
+ * @param f file to verify
+ * @return true if it has the correct header
+ * @throws IOException
+ */
+ private static boolean isValidCheckedStream(File f) throws IOException {
+ try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
+ // including the header and the last / bytes
+ // the snapshot should be at least 10 bytes
+ if (raf.length() < 10) {
+ return false;
+ }
+
+ raf.seek(raf.length() - 5);
+ byte bytes[] = new byte[5];
+ int readlen = 0;
+ int l;
+ while (readlen < 5 &&
+ (l = raf.read(bytes, readlen, bytes.length - readlen)) >= 0) {
+ readlen += l;
+ }
+ if (readlen != bytes.length) {
+ LOG.info("Invalid snapshot " + f.getName()
+ + ". too short, len = " + readlen + " bytes");
+ return false;
+ }
+ ByteBuffer bb = ByteBuffer.wrap(bytes);
+ int len = bb.getInt();
+ byte b = bb.get();
+ if (len != 1 || b != '/') {
+ LOG.info("Invalid snapshot " + f.getName() + ". len = " + len
+ + ", byte = " + (b & 0xff));
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/Util.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/Util.java
index d8dedb9..550260f 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/Util.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/Util.java
@@ -22,7 +22,6 @@ import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.io.Serializable;
import java.net.URI;
import java.nio.ByteBuffer;
@@ -42,7 +41,7 @@ import org.slf4j.LoggerFactory;
import org.apache.zookeeper.txn.TxnHeader;
/**
- * A collection of utility methods for dealing with file name parsing,
+ * A collection of utility methods for dealing with file name parsing,
* low level I/O file operations and marshalling/unmarshalling.
*/
public class Util {
@@ -59,11 +58,11 @@ public class Util {
return uri.replace('\\', '/');
}
/**
- * Given two directory files the method returns a well-formed
+ * Given two directory files the method returns a well-formed
* logfile provider URI. This method is for backward compatibility with the
* existing code that only supports logfile persistence and expects these
two
* parameters passed either on the command-line or in the configuration
file.
- *
+ *
* @param dataDir snapshot directory
* @param dataLogDir transaction log directory
* @return logfile provider URI
@@ -71,13 +70,13 @@ public class Util {
public static URI makeFileLoggerURL(File dataDir, File dataLogDir){
return
URI.create(makeURIString(dataDir.getPath(),dataLogDir.getPath(),null));
}
-
+
public static URI makeFileLoggerURL(File dataDir, File dataLogDir,String
convPolicy){
return
URI.create(makeURIString(dataDir.getPath(),dataLogDir.getPath(),convPolicy));
}
/**
- * Creates a valid transaction log file name.
+ * Creates a valid transaction log file name.
*
* @param zxid used as a file name suffix (extension)
* @return file name
@@ -88,17 +87,18 @@ public class Util {
/**
* Creates a snapshot file name.
- *
+ *
* @param zxid used as a suffix
* @return file name
*/
public static String makeSnapshotName(long zxid) {
- return FileSnap.SNAPSHOT_FILE_PREFIX + "." + Long.toHexString(zxid);
+ return FileSnap.SNAPSHOT_FILE_PREFIX + "." + Long.toHexString(zxid)
+ + SnapStream.getStreamMode().getFileExtension();
}
-
+
/**
* Extracts snapshot directory property value from the container.
- *
+ *
* @param props properties container
* @return file representing the snapshot directory
*/
@@ -108,24 +108,24 @@ public class Util {
/**
* Extracts transaction log directory property value from the container.
- *
+ *
* @param props properties container
* @return file representing the txn log directory
*/
public static File getLogDir(Properties props){
return new File(props.getProperty(LOG_DIR));
}
-
+
/**
* Extracts the value of the dbFormatConversion attribute.
- *
+ *
* @param props properties container
* @return value of the dbFormatConversion attribute
*/
public static String getFormatConversionPolicy(Properties props){
return props.getProperty(DB_FORMAT_CONV);
}
-
+
/**
* Extracts zxid from the file name. The file name should have been created
* using one of the {@link #makeLogName(long)} or {@link
#makeSnapshotName(long)}.
@@ -137,7 +137,7 @@ public class Util {
public static long getZxidFromName(String name, String prefix) {
long zxid = -1;
String nameParts[] = name.split("\\.");
- if (nameParts.length == 2 && nameParts[0].equals(prefix)) {
+ if (nameParts.length >= 2 && nameParts[0].equals(prefix)) {
try {
zxid = Long.parseLong(nameParts[1], 16);
} catch (NumberFormatException e) {
@@ -147,53 +147,6 @@ public class Util {
}
/**
- * Verifies that the file is a valid snapshot. Snapshot may be invalid if
- * it's incomplete as in a situation when the server dies while in the
process
- * of storing a snapshot. Any file that is not a snapshot is also
- * an invalid snapshot.
- *
- * @param f file to verify
- * @return true if the snapshot is valid
- * @throws IOException
- */
- public static boolean isValidSnapshot(File f) throws IOException {
- if (f==null || Util.getZxidFromName(f.getName(),
FileSnap.SNAPSHOT_FILE_PREFIX) == -1)
- return false;
-
- // Check for a valid snapshot
- try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
- // including the header and the last / bytes
- // the snapshot should be at least 10 bytes
- if (raf.length() < 10) {
- return false;
- }
- raf.seek(raf.length() - 5);
- byte bytes[] = new byte[5];
- int readlen = 0;
- int l;
- while (readlen < 5 &&
- (l = raf.read(bytes, readlen, bytes.length - readlen)) >=
0) {
- readlen += l;
- }
- if (readlen != bytes.length) {
- LOG.info("Invalid snapshot " + f
- + " too short, len = " + readlen);
- return false;
- }
- ByteBuffer bb = ByteBuffer.wrap(bytes);
- int len = bb.getInt();
- byte b = bb.get();
- if (len != 1 || b != '/') {
- LOG.info("Invalid snapshot " + f + " len = " + len
- + " byte = " + (b & 0xff));
- return false;
- }
- }
-
- return true;
- }
-
- /**
* Reads a transaction entry from the input archive.
* @param ia archive to read from
* @return null if the entry is corrupted or EOF has been reached; a buffer
@@ -215,11 +168,11 @@ public class Util {
}catch(EOFException e){}
return null;
}
-
+
/**
* Serializes transaction header and transaction data into a byte buffer.
- *
+ *
* @param hdr transaction header
* @param txn transaction data
* @return serialized transaction record
@@ -239,7 +192,7 @@ public class Util {
/**
* Write the serialized transaction record to the output archive.
- *
+ *
* @param oa output archive
* @param bytes serialized transaction record
* @throws IOException
@@ -249,8 +202,8 @@ public class Util {
oa.writeBuffer(bytes, "txnEntry");
oa.writeByte((byte) 0x42, "EOR"); // 'B'
}
-
-
+
+
/**
* Compare file file names of form "prefix.version". Sort order result
* returned in order of version.
@@ -274,7 +227,7 @@ public class Util {
return ascending ? result : -result;
}
}
-
+
/**
* Sort the list of files. Recency as determined by the version component
* of the file name.
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
index 2d7aad1..3ba3872 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.zookeeper.ZKTestCase;
import org.apache.zookeeper.server.persistence.FileTxnLog;
+import org.apache.zookeeper.server.persistence.SnapStream;
import org.apache.zookeeper.server.persistence.Util;
import org.apache.zookeeper.test.ClientBase;
import org.junit.Assert;
@@ -125,7 +126,7 @@ public class ZooKeeperServerTest extends ZKTestCase {
if (!f.exists()) {
f.createNewFile();
}
- Assert.assertFalse("Snapshot file size is greater than 9 bytes",
Util.isValidSnapshot(f));
+ Assert.assertFalse("Snapshot file size is greater than 9 bytes",
SnapStream.isValidSnapshot(f));
Assert.assertTrue("Can't delete file", f.delete());
} catch (IOException e) {
} finally {
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/SnapStreamTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/SnapStreamTest.java
new file mode 100644
index 0000000..3b8cb2a
--- /dev/null
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/SnapStreamTest.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+
+import org.apache.jute.BinaryInputArchive;
+import org.apache.jute.BinaryOutputArchive;
+import org.apache.jute.InputArchive;
+import org.apache.jute.OutputArchive;
+import org.apache.zookeeper.server.persistence.SnapStream.StreamMode;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.Assert;
+
+import static org.apache.zookeeper.test.ClientBase.createTmpDir;
+
+public class SnapStreamTest {
+
+ @After
+ public void tearDown() {
+ System.clearProperty(SnapStream.ZOOKEEPER_SHAPSHOT_STREAM_MODE);
+ SnapStream.setStreamMode(StreamMode.DEFAULT_MODE);
+ }
+
+ @Test
+ public void testStreamMode() {
+ Assert.assertEquals(StreamMode.CHECKED.getName(), "");
+ Assert.assertEquals(StreamMode.CHECKED.getFileExtension(), "");
+ Assert.assertEquals(StreamMode.CHECKED, StreamMode.fromString("name"));
+ Assert.assertEquals(StreamMode.GZIP.getName(), "gz");
+ Assert.assertEquals(StreamMode.GZIP.getFileExtension(), ".gz");
+ Assert.assertEquals(StreamMode.GZIP, StreamMode.fromString("gz"));
+ Assert.assertEquals(StreamMode.SNAPPY.getName(), "snappy");
+ Assert.assertEquals(StreamMode.SNAPPY.getFileExtension(), ".snappy");
+ Assert.assertEquals(StreamMode.SNAPPY,
StreamMode.fromString("snappy"));
+ }
+
+ @Test
+ public void testGetStreamMode() {
+ Assert.assertEquals(
+ "expected to return un-compressed stream",
+ StreamMode.CHECKED,
+ SnapStream.getStreamMode("snapshot.180000e3a2")
+ );
+ Assert.assertEquals(
+ "expected to return snappy stream",
+ StreamMode.SNAPPY,
+ SnapStream.getStreamMode("snapshot.180000e3a2.snappy")
+ );
+ Assert.assertEquals(
+ "expected to return gzip stream",
+ StreamMode.GZIP,
+ SnapStream.getStreamMode("snapshot.180000e3a2.gz")
+ );
+ }
+
+ @Test
+ public void testSerializeDeserializeWithChecked() throws IOException {
+ testSerializeDeserialize(StreamMode.CHECKED, "");
+ }
+
+ @Test
+ public void testSerializeDeserializeWithSNAPPY() throws IOException {
+ testSerializeDeserialize(StreamMode.SNAPPY, ".snappy");
+ }
+
+ @Test
+ public void testSerializeDeserializeWithGZIP() throws IOException {
+ testSerializeDeserialize(StreamMode.GZIP, ".gz");
+ }
+
+ private void testSerializeDeserialize(StreamMode mode, String fileSuffix)
throws IOException {
+ SnapStream.setStreamMode(mode);
+
+ // serialize with gzip stream
+ File tmpDir = createTmpDir();
+ File file = new File(tmpDir, "snapshot.180000e3a2" + fileSuffix);
+ CheckedOutputStream os = SnapStream.getOutputStream(file);
+ OutputArchive oa = BinaryOutputArchive.getArchive(os);
+ FileHeader header = new FileHeader(FileSnap.SNAP_MAGIC, 2, 1);
+ header.serialize(oa, "fileheader");
+ SnapStream.sealStream(os, oa);
+ os.flush();
+ os.close();
+
+ Assert.assertTrue(SnapStream.isValidSnapshot(file));
+
+ // deserialize with gzip stream
+ CheckedInputStream is = SnapStream.getInputStream(file);
+ InputArchive ia = BinaryInputArchive.getArchive(is);
+ FileHeader restoredHeader = new FileHeader();
+ restoredHeader.deserialize(ia, "fileheader");
+ Assert.assertEquals(
+ "magic not the same", restoredHeader, header);
+ SnapStream.checkSealIntegrity(is, ia);
+ }
+
+ private void checkInvalidSnapshot(String filename) throws IOException {
+ // set the output stream mode to CHECKED
+ SnapStream.setStreamMode(StreamMode.CHECKED);
+
+ // serialize to CHECKED file without magic header
+ File tmpDir = createTmpDir();
+ File file = new File(tmpDir, filename);
+ OutputStream os = SnapStream.getOutputStream(file);
+ os.write(1);
+ os.flush();
+ os.close();
+ Assert.assertFalse(SnapStream.isValidSnapshot(file));
+ }
+
+ @Test
+ public void testInvalidSnapshot() throws IOException {
+ Assert.assertFalse(SnapStream.isValidSnapshot(null));
+
+ checkInvalidSnapshot("snapshot.180000e3a2");
+ checkInvalidSnapshot("snapshot.180000e3a2.gz");
+ checkInvalidSnapshot("snapshot.180000e3a2.snappy");
+ }
+}
