This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new b9d4e75b feat(python): Add LZ4 decompression support to Python
bindings and Meson build (#917)
b9d4e75b is described below
commit b9d4e75bf165e55b0ca094b50d75c53ae98ff0af
Author: Dewey Dunnington <[email protected]>
AuthorDate: Thu Jul 30 17:27:52 2026 -0400
feat(python): Add LZ4 decompression support to Python bindings and Meson
build (#917)
Adds LZ4 support to the meson and Python builds.
```python
import io
import nanoarrow as na
import pyarrow as pa
buf = io.BytesIO()
batch = pa.record_batch({"x": range(1000)})
with pa.ipc.new_stream(buf, batch.schema,
options=pa.ipc.IpcWriteOptions(compression="lz4")) as w:
w.write_batch(batch)
buf.seek(0)
na.ArrayStream.from_readable(buf).read_all()
# nanoarrow.Array<non-nullable struct<x: int64>>[1000]
# {'x': 0}
# {'x': 1}
# {'x': 2}
# {'x': 3}
# {'x': 4}
# {'x': 5}
# {'x': 6}
# {'x': 7}
# {'x': 8}
# {'x': 9}
# ...and 990 more items
```
---
meson.build | 6 ++++++
meson_options.txt | 1 +
python/meson.build | 5 ++++-
python/src/nanoarrow/meson.build | 3 ++-
python/subprojects/lz4.wrap | 30 ++++++++++++++++++++++++++++++
python/tests/test_ipc.py | 30 ++++++++++++++++++++++++++++++
subprojects/lz4.wrap | 30 ++++++++++++++++++++++++++++++
7 files changed, 103 insertions(+), 2 deletions(-)
diff --git a/meson.build b/meson.build
index 7bc5bb36..bcffc0b4 100644
--- a/meson.build
+++ b/meson.build
@@ -125,6 +125,12 @@ if get_option('ipc').enabled()
ipc_lib_c_args += '-DNANOARROW_IPC_WITH_ZSTD'
endif
+ if get_option('ipc_with_lz4').enabled()
+ lz4_dep = dependency('liblz4')
+ ipc_lib_deps += lz4_dep
+ ipc_lib_c_args += '-DNANOARROW_IPC_WITH_LZ4'
+ endif
+
install_headers(
'src/nanoarrow/nanoarrow_ipc.h',
'src/nanoarrow/ipc/flatcc_generated.h',
diff --git a/meson_options.txt b/meson_options.txt
index 806cf940..d0dc2960 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -21,6 +21,7 @@ option('benchmarks', type: 'feature', description: 'Build
benchmarks')
option('apps', type: 'feature', description: 'Build utility applications')
option('ipc', type: 'feature', description: 'Build IPC libraries')
option('ipc_with_zstd', type: 'feature', description: 'Build IPC libraries
with ZSTD compression support')
+option('ipc_with_lz4', type: 'feature', description: 'Build IPC libraries with
LZ4 compression support')
option(
'integration_tests', type: 'feature',
description: 'Build cross-implementation Arrow integration tests',
diff --git a/python/meson.build b/python/meson.build
index a7e57c98..1873b640 100644
--- a/python/meson.build
+++ b/python/meson.build
@@ -26,15 +26,18 @@ project(
'warning_level=2',
'c_std=c99',
'default_library=static',
- 'force_fallback_for=zstd',
+ 'force_fallback_for=zstd,lz4',
# We need to set these options at the project default_option level
# due to https://github.com/mesonbuild/meson/issues/6728
'arrow-nanoarrow:ipc=enabled',
'arrow-nanoarrow:ipc_with_zstd=enabled',
+ 'arrow-nanoarrow:ipc_with_lz4=enabled',
'arrow-nanoarrow:device=enabled',
'arrow-nanoarrow:namespace=PythonPkg',
'arrow-nanoarrow:tests=disabled',
'zstd:bin_programs=false',
+ 'lz4:programs=false',
+ 'lz4:tests=false',
],
)
diff --git a/python/src/nanoarrow/meson.build b/python/src/nanoarrow/meson.build
index 16ab032e..c161fe15 100644
--- a/python/src/nanoarrow/meson.build
+++ b/python/src/nanoarrow/meson.build
@@ -17,6 +17,7 @@
flatcc_dep = dependency('flatcc')
zstd_dep = dependency('libzstd')
+lz4_dep = dependency('liblz4')
nanoarrow_proj = subproject('arrow-nanoarrow')
nanoarrow_dep = nanoarrow_proj.get_variable('nanoarrow_dep')
nanoarrow_ipc_dep = nanoarrow_proj.get_variable('nanoarrow_ipc_dep')
@@ -78,7 +79,7 @@ foreach cyf : cyfiles
if stem in ['_array', '_device']
cyfile_deps += [nanoarrow_device_dep]
elif stem == '_ipc_lib'
- cyfile_deps += [nanoarrow_ipc_dep, flatcc_dep, zstd_dep]
+ cyfile_deps += [nanoarrow_ipc_dep, flatcc_dep, zstd_dep, lz4_dep]
endif
py.extension_module(
diff --git a/python/subprojects/lz4.wrap b/python/subprojects/lz4.wrap
new file mode 100644
index 00000000..49f2812b
--- /dev/null
+++ b/python/subprojects/lz4.wrap
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+directory = lz4-1.10.0
+source_url = https://github.com/lz4/lz4/archive/v1.10.0.tar.gz
+source_filename = lz4-1.10.0.tgz
+source_hash = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
+patch_filename = lz4_1.10.0-2_patch.zip
+patch_url = https://wrapdb.mesonbuild.com/v2/lz4_1.10.0-2/get_patch
+patch_hash = ce568b4e1a7a593bb233f3ec97af88ceb850524cef802fe3f916b21d8a79b6b6
+source_fallback_url =
https://github.com/mesonbuild/wrapdb/releases/download/lz4_1.10.0-2/lz4-1.10.0.tgz
+wrapdb_version = 1.10.0-2
+
+[provide]
+liblz4 = liblz4_dep
diff --git a/python/tests/test_ipc.py b/python/tests/test_ipc.py
index 8b691637..b1fc013c 100644
--- a/python/tests/test_ipc.py
+++ b/python/tests/test_ipc.py
@@ -104,6 +104,18 @@ def test_ipc_stream_compressed_example():
assert array.child(0).to_pylist() == [0, 1, 2]
+def test_ipc_stream_compressed_lz4_example():
+ buf = io.BytesIO()
+ buf.write(_EXAMPLE_IPC_SCHEMA)
+ buf.write(COMPRESSED_BATCH_LZ4)
+ buf.seek(0)
+
+ with InputStream.from_readable(buf) as inp:
+ array = na.Array(inp)
+ assert len(array) == 3
+ assert array.child(0).to_pylist() == [0, 1, 2]
+
+
def test_ipc_stream_python_exception_on_read():
class ExtraordinarilyInconvenientFile:
def readinto(self, *args, **kwargs):
@@ -263,4 +275,22 @@ COMPRESSED_BATCH = bytearray([
0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02,
0x00, 0x00,
0x00, 0x00, 0x00, 0x00
])
+
+COMPRESSED_BATCH_LZ4 = bytearray([
+ 0xff, 0xff, 0xff, 0xff, 0x98, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x0c, 0x00, 0x18, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00,
0x0c, 0x00,
+ 0x0c, 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x1c, 0x00, 0x00, 0x00,
0x28, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00,
0x1c, 0x00,
+ 0x10, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x0c, 0x00, 0x00, 0x00,
0x48, 0x00,
+ 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
0x04, 0x00,
+ 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x04, 0x22, 0x4d, 0x18, 0x60, 0x40, 0x82, 0x0c, 0x00, 0x00, 0x80, 0x00,
0x00, 0x00,
+ 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00
+])
# fmt: on
diff --git a/subprojects/lz4.wrap b/subprojects/lz4.wrap
new file mode 100644
index 00000000..49f2812b
--- /dev/null
+++ b/subprojects/lz4.wrap
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+directory = lz4-1.10.0
+source_url = https://github.com/lz4/lz4/archive/v1.10.0.tar.gz
+source_filename = lz4-1.10.0.tgz
+source_hash = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
+patch_filename = lz4_1.10.0-2_patch.zip
+patch_url = https://wrapdb.mesonbuild.com/v2/lz4_1.10.0-2/get_patch
+patch_hash = ce568b4e1a7a593bb233f3ec97af88ceb850524cef802fe3f916b21d8a79b6b6
+source_fallback_url =
https://github.com/mesonbuild/wrapdb/releases/download/lz4_1.10.0-2/lz4-1.10.0.tgz
+wrapdb_version = 1.10.0-2
+
+[provide]
+liblz4 = liblz4_dep
