Author: dcreager
Date: Fri Jul 26 13:40:47 2013
New Revision: 1507309
URL: http://svn.apache.org/r1507309
Log:
AVRO-896. C: Snappy compression codec.
Like all other codecs, this is optional. It will only be activated if the
snappy C library is installed at build time. Contributed by Grisha Trubetskoy.
Added:
avro/trunk/lang/c/FindSnappy.cmake
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/c/CMakeLists.txt
avro/trunk/lang/c/src/codec.c
avro/trunk/lang/c/src/codec.h
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jul 26 13:40:47 2013
@@ -14,6 +14,8 @@ Trunk (not yet released)
AVRO-823: C#: Add data file support. (David McIntosh via cutting)
+ AVRO-896. C: Snappy compression codec. (Grisha Trubetskoy via dcreager)
+
IMPROVEMENTS
AVRO-1260. Ruby: Improve read performance. (Martin Kleppmann via cutting)
Modified: avro/trunk/lang/c/CMakeLists.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/CMakeLists.txt?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/lang/c/CMakeLists.txt (original)
+++ avro/trunk/lang/c/CMakeLists.txt Fri Jul 26 13:40:47 2013
@@ -20,6 +20,8 @@ cmake_minimum_required(VERSION 2.4)
project(AvroC)
enable_testing()
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR})
+
# Eliminates warning about linker paths when linking both zlib and
# liblzma.
cmake_policy(SET CMP0003 NEW)
@@ -125,6 +127,7 @@ include_directories(${AvroC_SOURCE_DIR}/
# Enable codecs
+
find_package(ZLIB)
if (ZLIB_FOUND)
set(ZLIB_PKG zlib)
@@ -136,6 +139,16 @@ else (ZLIB_FOUND)
message("Disabled deflate codec. zlib not found.")
endif (ZLIB_FOUND)
+find_package(Snappy)
+if (SNAPPY_FOUND AND ZLIB_FOUND) # Snappy borrows crc32 from zlib
+ set(SNAPPY_PKG libsnappy)
+ add_definitions(-DSNAPPY_CODEC)
+ message("Enabled snappy codec")
+else (SNAPPY_FOUND AND ZLIB_FOUND)
+ set (SNAPPY_PKG "")
+ message("Disabled snappy codec. libsnappy not found or zlib not found.")
+endif (SNAPPY_FOUND AND ZLIB_FOUND)
+
find_package(PkgConfig)
pkg_check_modules(LZMA liblzma)
if (LZMA_FOUND)
@@ -149,8 +162,8 @@ else (LZMA_FOUND)
message("Disabled lzma codec. liblzma not found.")
endif (LZMA_FOUND)
-set(CODEC_LIBRARIES ${ZLIB_LIBRARIES} ${LZMA_LIBRARIES})
-set(CODEC_PKG "@ZLIB_PKG@ @LZMA_PKG@")
+set(CODEC_LIBRARIES ${ZLIB_LIBRARIES} ${LZMA_LIBRARIES} ${SNAPPY_LIBRARIES})
+set(CODEC_PKG "@ZLIB_PKG@ @LZMA_PKG@ @SNAPPY_PKG@")
add_subdirectory(src)
Added: avro/trunk/lang/c/FindSnappy.cmake
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/FindSnappy.cmake?rev=1507309&view=auto
==============================================================================
--- avro/trunk/lang/c/FindSnappy.cmake (added)
+++ avro/trunk/lang/c/FindSnappy.cmake Fri Jul 26 13:40:47 2013
@@ -0,0 +1,36 @@
+# Tries to find Snappy headers and libraries.
+#
+# Usage of this module as follows:
+#
+# find_package(Snappy)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# SNAPPY_ROOT_DIR Set this variable to the root installation of
+# Snappy if the module has problems finding
+# the proper installation path.
+#
+# Variables defined by this module:
+#
+# SNAPPY_FOUND System has Snappy libs/headers
+# SNAPPY_LIBRARIES The Snappy libraries
+# SNAPPY_INCLUDE_DIR The location of Snappy headers
+
+find_path(SNAPPY_INCLUDE_DIR
+ NAMES snappy.h
+ HINTS ${SNAPPY_ROOT_DIR}/include)
+
+find_library(SNAPPY_LIBRARIES
+ NAMES snappy
+ HINTS ${SNAPPY_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Snappy DEFAULT_MSG
+ SNAPPY_LIBRARIES
+ SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(
+ SNAPPY_ROOT_DIR
+ SNAPPY_LIBRARIES
+ SNAPPY_INCLUDE_DIR)
Modified: avro/trunk/lang/c/src/codec.c
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/codec.c?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/lang/c/src/codec.c (original)
+++ avro/trunk/lang/c/src/codec.c Fri Jul 26 13:40:47 2013
@@ -16,6 +16,10 @@
*/
#include <string.h>
+#ifdef SNAPPY_CODEC
+#include <snappy-c.h>
+#include <byteswap.h>
+#endif
#ifdef DEFLATE_CODEC
#include <zlib.h>
#endif
@@ -71,6 +75,112 @@ static int reset_null(avro_codec_t c)
return 0;
}
+/* Snappy codec */
+
+#ifdef SNAPPY_CODEC
+
+static int
+codec_snappy(avro_codec_t codec)
+{
+ codec->name = "snappy";
+ codec->type = AVRO_CODEC_SNAPPY;
+ codec->block_size = 0;
+ codec->used_size = 0;
+ codec->block_data = NULL;
+ codec->codec_data = NULL;
+
+ return 0;
+}
+
+static int encode_snappy(avro_codec_t c, void * data, int64_t len)
+{
+ uint32_t crc;
+ size_t outlen = snappy_max_compressed_length(len);
+
+ if (!c->block_data) {
+ c->block_data = avro_malloc(outlen+4);
+ c->block_size = outlen+4;
+ } else if (c->block_size < (int64_t) (outlen+4)) {
+ c->block_data = avro_realloc(c->block_data, c->block_size,
(outlen+4));
+ c->block_size = outlen+4;
+ }
+
+ if (!c->block_data) {
+ avro_set_error("Cannot allocate memory for snappy");
+ return 1;
+ }
+
+ if (snappy_compress(data, len, c->block_data, &outlen) != SNAPPY_OK)
+ {
+ avro_set_error("Error compressing block with Snappy");
+ return 1;
+ }
+
+ crc = __bswap_32(crc32(0, data, len));
+ memcpy(c->block_data+outlen, &crc, 4);
+ c->used_size = outlen+4;
+
+ return 0;
+}
+
+static int decode_snappy(avro_codec_t c, void * data, int64_t len)
+{
+ uint32_t crc;
+ size_t outlen;
+
+ if (snappy_uncompressed_length(data, len-4, &outlen) != SNAPPY_OK) {
+ avro_set_error("Uncompressed length error in snappy");
+ return 1;
+ }
+
+ if (!c->block_data) {
+ c->block_data = avro_malloc(outlen);
+ c->block_size = outlen;
+ } else if ( (size_t)c->block_size < outlen) {
+ c->block_data = avro_realloc(c->block_data, c->block_size,
outlen);
+ c->block_size = outlen;
+ }
+
+ if (!c->block_data)
+ {
+ avro_set_error("Cannot allocate memory for snappy");
+ return 1;
+ }
+
+ if (snappy_uncompress(data, len-4, c->block_data, &outlen) !=
SNAPPY_OK)
+ {
+ avro_set_error("Error uncompressing block with Snappy");
+ return 1;
+ }
+
+ crc = __bswap_32(crc32(0, c->block_data, outlen));
+ if (memcmp(&crc, (char*)data+len-4, 4))
+ {
+ avro_set_error("CRC32 check failure uncompressing block with
Snappy");
+ return 1;
+ }
+
+ c->used_size = outlen;
+
+ return 0;
+}
+
+static int reset_snappy(avro_codec_t c)
+{
+ if (c->block_data) {
+ avro_free(c->block_data, c->block_size);
+ }
+
+ c->block_data = NULL;
+ c->block_size = 0;
+ c->used_size = 0;
+ c->codec_data = NULL;
+
+ return 0;
+}
+
+#endif // SNAPPY_CODEC
+
/* Deflate codec */
#ifdef DEFLATE_CODEC
@@ -403,6 +513,12 @@ int avro_codec(avro_codec_t codec, const
return codec_null(codec);
}
+#ifdef SNAPPY_CODEC
+ if (strcmp("snappy", type) == 0) {
+ return codec_snappy(codec);
+ }
+#endif
+
#ifdef DEFLATE_CODEC
if (strcmp("deflate", type) == 0) {
return codec_deflate(codec);
@@ -429,6 +545,10 @@ int avro_codec_encode(avro_codec_t c, vo
{
case AVRO_CODEC_NULL:
return encode_null(c, data, len);
+#ifdef SNAPPY_CODEC
+ case AVRO_CODEC_SNAPPY:
+ return encode_snappy(c, data, len);
+#endif
#ifdef DEFLATE_CODEC
case AVRO_CODEC_DEFLATE:
return encode_deflate(c, data, len);
@@ -448,6 +568,10 @@ int avro_codec_decode(avro_codec_t c, vo
{
case AVRO_CODEC_NULL:
return decode_null(c, data, len);
+#ifdef SNAPPY_CODEC
+ case AVRO_CODEC_SNAPPY:
+ return decode_snappy(c, data, len);
+#endif
#ifdef DEFLATE_CODEC
case AVRO_CODEC_DEFLATE:
return decode_deflate(c, data, len);
@@ -467,6 +591,10 @@ int avro_codec_reset(avro_codec_t c)
{
case AVRO_CODEC_NULL:
return reset_null(c);
+#ifdef SNAPPY_CODEC
+ case AVRO_CODEC_SNAPPY:
+ return reset_snappy(c);
+#endif
#ifdef DEFLATE_CODEC
case AVRO_CODEC_DEFLATE:
return reset_deflate(c);
Modified: avro/trunk/lang/c/src/codec.h
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/codec.h?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/lang/c/src/codec.h (original)
+++ avro/trunk/lang/c/src/codec.h Fri Jul 26 13:40:47 2013
@@ -29,7 +29,8 @@ extern "C" {
enum avro_codec_type_t {
AVRO_CODEC_NULL,
AVRO_CODEC_DEFLATE,
- AVRO_CODEC_LZMA
+ AVRO_CODEC_LZMA,
+ AVRO_CODEC_SNAPPY
};
typedef enum avro_codec_type_t avro_codec_type_t;
