This is an automated email from the ASF dual-hosted git repository.
bkietz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 242f98c1f0 GH-43680: [Integration] Unskip nanoarrow in IPC integration
tests (#43715)
242f98c1f0 is described below
commit 242f98c1f0a79db6843d24e5052790f17aa76ae2
Author: Benjamin Kietzman <[email protected]>
AuthorDate: Tue Sep 24 11:58:57 2024 -0500
GH-43680: [Integration] Unskip nanoarrow in IPC integration tests (#43715)
### Rationale for this change
Nanoarrow can now read and write IPC files as of
https://github.com/apache/arrow-nanoarrow/pull/585 so it should no longer be
skipped as a producer/consumer
### What changes are included in this PR?
Nanoarrow's tester is updated to point to the new integration executable
and to report nanoarrow as a consumer/producer of IPC files.
Notably the `null_trivial` case is skipped even though nanoarrow nominally
supports it since it represents a corner case in which nanoarrow's flatbuffers
library will not accept some vectors produced by other flatbuffers libraries
https://github.com/dvidelabs/flatcc/issues/287
### Are these changes tested?
Yes
### Are there any user-facing changes?
No
* GitHub Issue: #43680
Lead-authored-by: Benjamin Kietzman <[email protected]>
Co-authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Benjamin Kietzman <[email protected]>
---
ci/scripts/nanoarrow_build.sh | 2 +-
dev/archery/archery/integration/datagen.py | 11 +++++-
.../archery/integration/tester_nanoarrow.py | 39 ++++++++++++++++++----
3 files changed, 44 insertions(+), 8 deletions(-)
diff --git a/ci/scripts/nanoarrow_build.sh b/ci/scripts/nanoarrow_build.sh
index 1612b9a2d0..ed5cf6298f 100755
--- a/ci/scripts/nanoarrow_build.sh
+++ b/ci/scripts/nanoarrow_build.sh
@@ -46,7 +46,7 @@ set -x
mkdir -p ${build_dir}
pushd ${build_dir}
-cmake ${source_dir} -DNANOARROW_BUILD_INTEGRATION_TESTS=ON
+cmake ${source_dir} -DNANOARROW_IPC=ON -DNANOARROW_BUILD_INTEGRATION_TESTS=ON
cmake --build .
popd
diff --git a/dev/archery/archery/integration/datagen.py
b/dev/archery/archery/integration/datagen.py
index f63aa0d95a..970fe2e16b 100644
--- a/dev/archery/archery/integration/datagen.py
+++ b/dev/archery/archery/integration/datagen.py
@@ -1914,33 +1914,42 @@ def get_generated_json_files(tempdir=None):
generate_duplicate_fieldnames_case()
.skip_tester('JS'),
- generate_dictionary_case(),
+ generate_dictionary_case()
+ # TODO(https://github.com/apache/arrow-nanoarrow/issues/622)
+ .skip_tester('nanoarrow'),
generate_dictionary_unsigned_case()
+ .skip_tester('nanoarrow')
.skip_tester('Java'), # TODO(ARROW-9377)
generate_nested_dictionary_case()
+ # TODO(https://github.com/apache/arrow-nanoarrow/issues/622)
+ .skip_tester('nanoarrow')
.skip_tester('Java'), # TODO(ARROW-7779)
generate_run_end_encoded_case()
.skip_tester('C#')
.skip_tester('Java')
.skip_tester('JS')
+ # TODO(https://github.com/apache/arrow-nanoarrow/issues/618)
.skip_tester('nanoarrow')
.skip_tester('Rust'),
generate_binary_view_case()
.skip_tester('JS')
+ # TODO(https://github.com/apache/arrow-nanoarrow/issues/618)
.skip_tester('nanoarrow')
.skip_tester('Rust'),
generate_list_view_case()
.skip_tester('C#') # Doesn't support large list views
.skip_tester('JS')
+ # TODO(https://github.com/apache/arrow-nanoarrow/issues/618)
.skip_tester('nanoarrow')
.skip_tester('Rust'),
generate_extension_case()
+ .skip_tester('nanoarrow')
# TODO: ensure the extension is registered in the C++ entrypoint
.skip_format(SKIP_C_SCHEMA, 'C++')
.skip_format(SKIP_C_ARRAY, 'C++'),
diff --git a/dev/archery/archery/integration/tester_nanoarrow.py
b/dev/archery/archery/integration/tester_nanoarrow.py
index 30ff1bb6e5..5af469d7a1 100644
--- a/dev/archery/archery/integration/tester_nanoarrow.py
+++ b/dev/archery/archery/integration/tester_nanoarrow.py
@@ -20,6 +20,7 @@ import os
from . import cdata
from .tester import Tester, CDataExporter, CDataImporter
+from .util import run_cmd, log
from ..utils.source import ARROW_ROOT_DEFAULT
@@ -32,10 +33,14 @@ _INTEGRATION_DLL = os.path.join(
_NANOARROW_PATH, "libnanoarrow_c_data_integration" + cdata.dll_suffix
)
+_INTEGRATION_EXE = os.path.join(
+ _NANOARROW_PATH, "nanoarrow_ipc_integration"
+)
+
class NanoarrowTester(Tester):
- PRODUCER = False
- CONSUMER = False
+ PRODUCER = True
+ CONSUMER = True
FLIGHT_SERVER = False
FLIGHT_CLIENT = False
C_DATA_SCHEMA_EXPORTER = True
@@ -45,17 +50,39 @@ class NanoarrowTester(Tester):
name = "nanoarrow"
+ def _run(self, arrow_path, json_path, command, quirks):
+ env = {
+ 'ARROW_PATH': arrow_path,
+ 'JSON_PATH': json_path,
+ 'COMMAND': command,
+ **{
+ f'QUIRK_{q}': "1"
+ for q in quirks or ()
+ },
+ }
+
+ if self.debug:
+ log(f'{_INTEGRATION_EXE} {env}')
+
+ run_cmd([_INTEGRATION_EXE], env=env)
+
def validate(self, json_path, arrow_path, quirks=None):
- raise NotImplementedError()
+ return self._run(arrow_path, json_path, 'VALIDATE', quirks)
def json_to_file(self, json_path, arrow_path):
- raise NotImplementedError()
+ return self._run(arrow_path, json_path, 'JSON_TO_ARROW', quirks=None)
def stream_to_file(self, stream_path, file_path):
- raise NotImplementedError()
+ self.run_shell_command([_INTEGRATION_EXE, '<', stream_path], env={
+ 'COMMAND': 'STREAM_TO_FILE',
+ 'ARROW_PATH': file_path,
+ })
def file_to_stream(self, file_path, stream_path):
- raise NotImplementedError()
+ self.run_shell_command([_INTEGRATION_EXE, '>', stream_path], env={
+ 'COMMAND': 'FILE_TO_STREAM',
+ 'ARROW_PATH': file_path,
+ })
def make_c_data_exporter(self):
return NanoarrowCDataExporter(self.debug, self.args)
