This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 738b821f fix(ci): Fix build and test of nanoarrow on centos7 and s390x
(#576)
738b821f is described below
commit 738b821f9f68e25161913e97370426c473948e39
Author: Dewey Dunnington <[email protected]>
AuthorDate: Tue Aug 6 16:27:50 2024 -0300
fix(ci): Fix build and test of nanoarrow on centos7 and s390x (#576)
Noted in the weekly verification run:
https://github.com/apache/arrow-nanoarrow/actions/runs/10232042714/job/28308701770
I think there were a three issues:
- The bundling setup used a feature of the command execution not
available in stock `cmake3` on centos7
- Some C++17 (or unsupported gcc4.8 C++11) in some of the newer IPC
additions
- IPC tests were failing on big endian (
https://github.com/apache/arrow-nanoarrow/actions/runs/10232042714/job/28308702337#step:4:2640
)
Reproducible via:
```shell
export NANOARROW_PLATFORM=centos7
docker compose run -e TEST_PYTHON=0 -e TEST_R=0 --rm verify
```
```shell
export NANOARROW_PLATFORM=alpine
export NANOARROW_ARCH=s390x
docker compose run -e TEST_DEFAULT=0 -e TEST_C=1 --rm verify
```
---
.github/workflows/verify.yaml | 2 +-
CMakeLists.txt | 2 +-
src/nanoarrow/common/array_test.cc | 23 ++++++++++++-----------
src/nanoarrow/ipc/decoder_test.cc | 10 ++++------
src/nanoarrow/ipc/encoder.c | 7 +++++--
src/nanoarrow/ipc/writer_test.cc | 12 ++++++------
6 files changed, 29 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
index d0e54da4..0ff5b796 100644
--- a/.github/workflows/verify.yaml
+++ b/.github/workflows/verify.yaml
@@ -71,7 +71,7 @@ jobs:
use-public-rspm: true
windows-path-include-rtools: false
windows-path-include-mingw: false
- - uses: r-lib/actions/setup-r-dependencies@v2
+ - uses:
r-lib/actions/setup-r-dependencies@f4937e0dc26f9b99c969cd3e4ca943b576e7f991
with:
needs: check
working-directory: src/r
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0aacb2fc..e73779a9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -106,7 +106,7 @@ if(NANOARROW_BUNDLE)
execute_process(COMMAND ${Python3_EXECUTABLE}
"${CMAKE_CURRENT_SOURCE_DIR}/ci/scripts/bundle.py"
"--output-dir" "${CMAKE_BINARY_DIR}/bundled"
- ${NANOARROW_BUNDLE_ARGS} COMMAND_ECHO STDERR)
+ ${NANOARROW_BUNDLE_ARGS})
set(NANOARROW_BUILD_INCLUDE_DIR "${CMAKE_BINARY_DIR}/bundled/include")
set(NANOARROW_IPC_BUILD_SOURCES
"${CMAKE_BINARY_DIR}/bundled/src/nanoarrow_ipc.c")
diff --git a/src/nanoarrow/common/array_test.cc
b/src/nanoarrow/common/array_test.cc
index e55bb6a5..030c5699 100644
--- a/src/nanoarrow/common/array_test.cc
+++ b/src/nanoarrow/common/array_test.cc
@@ -1820,9 +1820,9 @@ TEST(ArrayTest, ArrayViewTestComputeNullCount) {
struct ArrowError error;
int32_t values[] = {17, 87, 23, 53};
- uint8_t all_valid = 0b1111'1111;
- uint8_t all_null = 0b0000'0000;
- uint8_t half_valid = 0b1010'1010;
+ uint8_t all_valid = 0b11111111;
+ uint8_t all_null = 0b00000000;
+ uint8_t half_valid = 0b10101010;
uint8_t* all_valid_because_missing = nullptr;
const void* buffers[2];
@@ -1838,19 +1838,20 @@ TEST(ArrayTest, ArrayViewTestComputeNullCount) {
array->dictionary = nullptr;
array->release = [](struct ArrowArray*) {};
- for (auto [buffer, null_count] : {
- std::pair{&all_valid, int64_t(0)},
- std::pair{&all_null, array->length},
- std::pair{&half_valid, array->length / 2},
- std::pair{all_valid_because_missing, int64_t(0)},
+ using test_pair = std::pair<uint8_t*, int64_t>;
+ for (const auto& item : {
+ test_pair{&all_valid, int64_t(0)},
+ test_pair{&all_null, array->length},
+ test_pair{&half_valid, array->length / 2},
+ test_pair{all_valid_because_missing, int64_t(0)},
}) {
- array->null_count = null_count;
- buffers[0] = buffer;
+ array->null_count = item.second;
+ buffers[0] = item.first;
nanoarrow::UniqueArrayView array_view;
ArrowArrayViewInitFromType(array_view.get(), NANOARROW_TYPE_INT32);
EXPECT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), &error),
NANOARROW_OK)
<< error.message;
- EXPECT_EQ(ArrowArrayViewComputeNullCount(array_view.get()), null_count);
+ EXPECT_EQ(ArrowArrayViewComputeNullCount(array_view.get()), item.second);
}
array->length = 0;
diff --git a/src/nanoarrow/ipc/decoder_test.cc
b/src/nanoarrow/ipc/decoder_test.cc
index 45f4f53b..1560a835 100644
--- a/src/nanoarrow/ipc/decoder_test.cc
+++ b/src/nanoarrow/ipc/decoder_test.cc
@@ -493,12 +493,10 @@ std::string ArrowSchemaMetadataToString(const char*
metadata) {
}
std::string ArrowSchemaToString(const struct ArrowSchema* schema) {
- std::string out;
- size_t n = 1024;
- while (out.size() < n) {
- out.resize(n);
- n = ArrowSchemaToString(schema, out.data(), out.size(),
/*recursive=*/false);
- }
+ int64_t n = ArrowSchemaToString(schema, nullptr, 0, /*recursive=*/false);
+ std::vector<char> out_vec(n, '\0');
+ ArrowSchemaToString(schema, out_vec.data(), n, /*recursive=*/false);
+ std::string out(out_vec.data(), out_vec.size());
std::string metadata = ArrowSchemaMetadataToString(schema->metadata);
if (!metadata.empty()) {
diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c
index fcfd997c..4eebe36e 100644
--- a/src/nanoarrow/ipc/encoder.c
+++ b/src/nanoarrow/ipc/encoder.c
@@ -401,8 +401,11 @@ ArrowErrorCode ArrowIpcEncoderEncodeSchema(struct
ArrowIpcEncoder* encoder,
FLATCC_RETURN_UNLESS_0(Message_header_Schema_start(builder));
- FLATCC_RETURN_UNLESS_0(
- Schema_endianness_add(builder,
(ns(Endianness_enum_t))ArrowIpcSystemEndianness()));
+ if (ArrowIpcSystemEndianness() == NANOARROW_IPC_ENDIANNESS_LITTLE) {
+ FLATCC_RETURN_UNLESS_0(Schema_endianness_add(builder,
ns(Endianness_Little)));
+ } else {
+ FLATCC_RETURN_UNLESS_0(Schema_endianness_add(builder, ns(Endianness_Big)));
+ }
FLATCC_RETURN_UNLESS_0(Schema_fields_start(builder));
NANOARROW_RETURN_NOT_OK(ArrowIpcEncodeFields(builder, schema,
diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc
index d0c344f8..1730bd4d 100644
--- a/src/nanoarrow/ipc/writer_test.cc
+++ b/src/nanoarrow/ipc/writer_test.cc
@@ -21,8 +21,6 @@
#include "nanoarrow/nanoarrow_ipc.hpp"
-using nanoarrow::literals::operator""_asv;
-
TEST(NanoarrowIpcWriter, OutputStreamBuffer) {
struct ArrowError error;
@@ -47,9 +45,10 @@ TEST(NanoarrowIpcWriter, OutputStreamBuffer) {
EXPECT_EQ(output->size_bytes, header.size() + 4 * message.size());
- std::string output_str(output->size_bytes, '\0');
+ std::vector<char> output_str(output->size_bytes, '\0');
memcpy(output_str.data(), output->data, output->size_bytes);
- EXPECT_EQ(output_str, header + message + message + message + message);
+ EXPECT_EQ(std::string(output_str.data(), output_str.size()),
+ header + message + message + message + message);
}
TEST(NanoarrowIpcWriter, OutputStreamFile) {
@@ -81,10 +80,11 @@ TEST(NanoarrowIpcWriter, OutputStreamFile) {
// Read back the whole file
fseek(file_ptr, 0, SEEK_END);
- std::string buffer(static_cast<size_t>(ftell(file_ptr)), '\0');
+ std::vector<char> buffer(static_cast<size_t>(ftell(file_ptr)), '\0');
rewind(file_ptr);
ASSERT_EQ(fread(buffer.data(), 1, buffer.size(), file_ptr), buffer.size());
EXPECT_EQ(buffer.size(), 6 + 4 * message.size());
- EXPECT_EQ(buffer, "HELLO " + message + message + message + message);
+ EXPECT_EQ(std::string(buffer.data(), buffer.size()),
+ "HELLO " + message + message + message + message);
}
