svn commit: r1095208 - in /avro/trunk: CHANGES.txt doc/src/content/xdocs/spec.xml lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java

[cutting]

Author: cutting
Date: Tue Apr 19 20:52:12 2011
New Revision: 1095208

URL: http://svn.apache.org/viewvc?rev=1095208&view=rev
Log:
AVRO-798. Add checksum to Snappy compressed blocks.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/doc/src/content/xdocs/spec.xml
    
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java

Modified: avro/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1095208&r1=1095207&r2=1095208&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Tue Apr 19 20:52:12 2011
@@ -50,6 +50,8 @@ Avro 1.5.1 (unreleased)
     'fromtext' command.  Also made some performance improvements, bug
     fixes and added tests for this command. (cutting)
 
+    AVRO-798. Add checksum to Snappy compressed blocks. (cutting)
+
   BUG FIXES
 
     AVRO-786. Java: Fix equals() to work on objects containing maps. (cutting)

Modified: avro/trunk/doc/src/content/xdocs/spec.xml
URL: 
http://svn.apache.org/viewvc/avro/trunk/doc/src/content/xdocs/spec.xml?rev=1095208&r1=1095207&r2=1095208&view=diff
==============================================================================
--- avro/trunk/doc/src/content/xdocs/spec.xml (original)
+++ avro/trunk/doc/src/content/xdocs/spec.xml Tue Apr 19 20:52:12 2011
@@ -701,7 +701,8 @@
           <title>snappy</title>
           <p>The "snappy" codec uses
             Google's <a href="http://code.google.com/p/snappy/";>Snappy</a>
-            compression library.</p>
+            compression library.  Each compressed block is followed
+            by its 4-byte, big-endian CRC32 checksum.</p>
         </section>
       </section>
     </section>

Modified: 
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java
URL: 
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java?rev=1095208&r1=1095207&r2=1095208&view=diff
==============================================================================
--- 
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java 
(original)
+++ 
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java 
Tue Apr 19 20:52:12 2011
@@ -19,19 +19,19 @@ package org.apache.avro.file;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
 
 import org.xerial.snappy.Snappy;
 import org.xerial.snappy.SnappyException;
 
 /** * Implements Snappy compression and decompression. */
 class SnappyCodec extends Codec {
-
-  private static final SnappyCodec INSTANCE = new SnappyCodec();
+  private CRC32 crc32 = new CRC32();
 
   static class Option extends CodecFactory {
     @Override
     protected Codec createInstance() {
-      return INSTANCE;
+      return new SnappyCodec();
     }
   }
 
@@ -43,10 +43,15 @@ class SnappyCodec extends Codec {
   ByteBuffer compress(ByteBuffer in) throws IOException {
     try { 
       ByteBuffer out =
-        ByteBuffer.allocate(Snappy.maxCompressedLength(in.remaining()));
+        ByteBuffer.allocate(Snappy.maxCompressedLength(in.remaining())+4);
       int size = Snappy.compress(in.array(), in.position(), in.remaining(),
                                  out.array(), 0);
-      out.limit(size);
+      crc32.reset();
+      crc32.update(in.array(), in.position(), in.remaining());
+      out.putInt(size, (int)crc32.getValue());
+
+      out.limit(size+4);
+
       return out;
     } catch (SnappyException e) {
       throw new IOException(e);
@@ -57,10 +62,16 @@ class SnappyCodec extends Codec {
   ByteBuffer decompress(ByteBuffer in) throws IOException {
     try { 
       ByteBuffer out = ByteBuffer.allocate
-        (Snappy.uncompressedLength(in.array(), in.position(), in.remaining()));
-      int size = Snappy.uncompress(in.array(), in.position(), in.remaining(),
+        (Snappy.uncompressedLength(in.array(),in.position(),in.remaining()-4));
+      int size = Snappy.uncompress(in.array(),in.position(),in.remaining()-4,
                                    out.array(), 0);
       out.limit(size);
+
+      crc32.reset();
+      crc32.update(out.array(), 0, size);
+      if (in.getInt(in.limit()-4) != (int)crc32.getValue())
+        throw new IOException("Checksum failure");
+
       return out;
     } catch (SnappyException e) {
       throw new IOException(e);