This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 96a7fb0 AVRO-2426: Add zstd support to Python2 bindings (#556)
96a7fb0 is described below
commit 96a7fb08e74cb7be1c63ab9c73ce0b25b1abfed1
Author: Kengo Seki <[email protected]>
AuthorDate: Fri Jun 28 21:24:03 2019 +0900
AVRO-2426: Add zstd support to Python2 bindings (#556)
* AVRO-2426: Add zstd support to Python2 bindings
* AVRO-2426: Add zstd support to Python2 bindings
Fix superfluous else statement
---
lang/py/src/avro/datafile.py | 22 ++++++++++++++++++++++
lang/py/test/test_datafile.py | 5 +++++
2 files changed, 27 insertions(+)
diff --git a/lang/py/src/avro/datafile.py b/lang/py/src/avro/datafile.py
index c85a896..6287312 100644
--- a/lang/py/src/avro/datafile.py
+++ b/lang/py/src/avro/datafile.py
@@ -28,6 +28,11 @@ try:
has_snappy = True
except ImportError:
has_snappy = False
+try:
+ import zstandard as zstd
+ has_zstandard = True
+except ImportError:
+ has_zstandard = False
#
# Constants
#
@@ -47,6 +52,8 @@ META_SCHEMA = schema.parse("""\
VALID_CODECS = ['null', 'deflate']
if has_snappy:
VALID_CODECS.append('snappy')
+if has_zstandard:
+ VALID_CODECS.append('zstandard')
VALID_ENCODINGS = ['binary'] # not used yet
CODEC_KEY = "avro.codec"
@@ -170,6 +177,9 @@ class DataFileWriter(object):
elif self.get_meta(CODEC_KEY) == 'snappy':
compressed_data = snappy.compress(uncompressed_data)
compressed_data_length = len(compressed_data) + 4 # crc32
+ elif self.get_meta(CODEC_KEY) == 'zstandard':
+ compressed_data = zstd.ZstdCompressor().compress(uncompressed_data)
+ compressed_data_length = len(compressed_data)
else:
fail_msg = '"%s" codec is not supported.' % self.get_meta(CODEC_KEY)
raise DataFileException(fail_msg)
@@ -331,6 +341,18 @@ class DataFileReader(object):
uncompressed = snappy.decompress(data)
self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
self.raw_decoder.check_crc32(uncompressed);
+ elif self.codec == 'zstandard':
+ length = self.raw_decoder.read_long()
+ data = self.raw_decoder.read(length)
+ uncompressed = bytearray()
+ dctx = zstd.ZstdDecompressor()
+ with dctx.stream_reader(StringIO(data)) as reader:
+ while True:
+ chunk = reader.read(16384)
+ if not chunk:
+ break
+ uncompressed.extend(chunk)
+ self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
else:
raise DataFileException("Unknown codec: %r" % self.codec)
diff --git a/lang/py/test/test_datafile.py b/lang/py/test/test_datafile.py
index 102d9b0..d0fc87f 100644
--- a/lang/py/test/test_datafile.py
+++ b/lang/py/test/test_datafile.py
@@ -60,6 +60,11 @@ try:
CODECS_TO_VALIDATE += ('snappy',)
except ImportError:
print 'Snappy not present, will skip testing it.'
+try:
+ import zstandard
+ CODECS_TO_VALIDATE += ('zstandard',)
+except ImportError:
+ print 'Zstandard not present, will skip testing it.'
# TODO(hammer): clean up written files with ant, not os.remove
class TestDataFile(unittest.TestCase):
