This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new a53e70b feat: Add Array/Schema/ArrayStream comparison utility to
testing helpers (#330)
a53e70b is described below
commit a53e70b2a83d840adc5b9392b2a5c63ad0958e90
Author: Dewey Dunnington <[email protected]>
AuthorDate: Thu Dec 7 14:50:15 2023 -0400
feat: Add Array/Schema/ArrayStream comparison utility to testing helpers
(#330)
This PR used a small demo utility (
https://gist.github.com/paleolimbot/ec2a2067198f0de1901c107c783d3b26 )
to run its own golden file tests against the arrow-testing repository
and make the necessary updates to the testing utility functions so that
those tests actually run! Notably:
- The testing JSON in those files omits `metadata` instead of writing
`"metadata": null`. I also dropped `"metadata": null` from nanoarrow's
JSON output to shorten the output a little.
- The testing JSON in those files omits `"bitWidth: 128` for decimal128
types. I kept this in nanoarrow's output but had to update the reader to
allow for this key to be missing.
- The testing JSON in those files always writes the data buffer for NULL
slots as zeroed memory, whereas the IPC in the testing repo often
contains non-zeroed memory. I adjusted nanoarrow's output to always
write the zeroed-memory version (which ensures that two arrays
serialized to testing JSON using nanoarrow's writer can be considered
equal for the purposes of integration testing).
The main addition required for these tests to pass was a "comparison"
utility. The strategy here is basically to compare the JSON output from
nanoarrow's JSON writer except at the very top level. This is not
suitable for general-purpose comparison but works within the constraints
of integration testing and generates reasonably nice output in the cases
of failure.
The result of running a demo utility (
https://gist.github.com/paleolimbot/ec2a2067198f0de1901c107c783d3b26 )
against all files in the arrow-testing repo's 1.0.0 integration
collection results in a lot of passed tests (with failing tests
basically just for unsupported types like date/time and dictionary).
```
dewey@Deweys-Mac-mini nanoarrow_ipc % export
NANOARROW_ARROW_TESTING_DIR=/Users/dewey/Desktop/rscratch/arrow-testing
dewey@Deweys-Mac-mini nanoarrow_ipc % cd out/build/user-local
dewey@Deweys-Mac-mini user-local %
../../../scripts/test_integration_test_util.sh
[PASS] generated_primitive_zerolength.stream --check
generated_primitive_zerolength.stream
[PASS] generated_primitive_zerolength.json.gz --check
generated_primitive_zerolength.json.gz
[PASS] generated_primitive_zerolength.json.gz --check
generated_primitive_zerolength.stream
[PASS] generated_null.stream --check generated_null.stream
[PASS] generated_null.json.gz --check generated_null.json.gz
[PASS] generated_null.json.gz --check generated_null.stream
[PASS] generated_map_non_canonical.stream --check
generated_map_non_canonical.stream
[PASS] generated_map_non_canonical.json.gz --check
generated_map_non_canonical.json.gz
[PASS] generated_map_non_canonical.json.gz --check
generated_map_non_canonical.stream
[SKIP] generated_dictionary_unsigned.json.gz
[PASS] generated_map.stream --check generated_map.stream
[PASS] generated_map.json.gz --check generated_map.json.gz
[PASS] generated_map.json.gz --check generated_map.stream
[PASS] generated_primitive.stream --check generated_primitive.stream
[PASS] generated_primitive.json.gz --check generated_primitive.json.gz
[PASS] generated_primitive.json.gz --check generated_primitive.stream
[PASS] generated_duplicate_fieldnames.stream --check
generated_duplicate_fieldnames.stream
[PASS] generated_duplicate_fieldnames.json.gz --check
generated_duplicate_fieldnames.json.gz
[PASS] generated_duplicate_fieldnames.json.gz --check
generated_duplicate_fieldnames.stream
writer_.WriteColumn(ss, schema, expected) failed with errno 45
*
/Users/dewey/Desktop/rscratch/arrow-nanoarrow/src/nanoarrow/nanoarrow_testing.hpp:1911
[FAIL] generated_decimal256.stream --check generated_decimal256.stream
-> Column 'f0' storage type decimal256 DATA buffer not supported
[FAIL] generated_decimal256.json.gz --check generated_decimal256.json.gz
-> Column 'f0' storage type decimal256 DATA buffer not supported
[FAIL] generated_decimal256.json.gz --check generated_decimal256.stream
writer_.WriteColumn(ss, schema, expected) failed with errno 45
*
/Users/dewey/Desktop/rscratch/arrow-nanoarrow/src/nanoarrow/nanoarrow_testing.hpp:1911
[FAIL] generated_decimal.stream --check generated_decimal.stream
-> Column 'f0' storage type decimal128 DATA buffer not supported
[FAIL] generated_decimal.json.gz --check generated_decimal.json.gz
-> Column 'f0' storage type decimal128 DATA buffer not supported
[FAIL] generated_decimal.json.gz --check generated_decimal.stream
[PASS] generated_custom_metadata.stream --check
generated_custom_metadata.stream
[PASS] generated_custom_metadata.json.gz --check
generated_custom_metadata.json.gz
[PASS] generated_custom_metadata.json.gz --check
generated_custom_metadata.stream
[PASS] generated_null_trivial.stream --check generated_null_trivial.stream
[PASS] generated_null_trivial.json.gz --check generated_null_trivial.json.gz
[PASS] generated_null_trivial.json.gz --check generated_null_trivial.stream
writer_.WriteField(ss, expected) failed with errno 45
*
/Users/dewey/Desktop/rscratch/arrow-nanoarrow/src/nanoarrow/nanoarrow_testing.hpp:1865
[FAIL] generated_interval.stream --check generated_interval.stream
Unsupported Type name: 'duration'
[FAIL] generated_interval.json.gz --check generated_interval.json.gz
Unsupported Type name: 'duration'
[FAIL] generated_interval.json.gz --check generated_interval.stream
[PASS] generated_union.stream --check generated_union.stream
[PASS] generated_union.json.gz --check generated_union.json.gz
[PASS] generated_union.json.gz --check generated_union.stream
[PASS] generated_nested.stream --check generated_nested.stream
[PASS] generated_nested.json.gz --check generated_nested.json.gz
[PASS] generated_nested.json.gz --check generated_nested.stream
[PASS] generated_recursive_nested.stream --check
generated_recursive_nested.stream
[PASS] generated_recursive_nested.json.gz --check
generated_recursive_nested.json.gz
[PASS] generated_recursive_nested.json.gz --check
generated_recursive_nested.stream
[SKIP] generated_dictionary.json.gz
writer_.WriteField(ss, expected) failed with errno 45
*
/Users/dewey/Desktop/rscratch/arrow-nanoarrow/src/nanoarrow/nanoarrow_testing.hpp:1865
[FAIL] generated_datetime.stream --check generated_datetime.stream
Unsupported Type name: 'date'
[FAIL] generated_datetime.json.gz --check generated_datetime.json.gz
Unsupported Type name: 'date'
[FAIL] generated_datetime.json.gz --check generated_datetime.stream
[PASS] generated_primitive_large_offsets.stream --check
generated_primitive_large_offsets.stream
[PASS] generated_primitive_large_offsets.json.gz --check
generated_primitive_large_offsets.json.gz
[PASS] generated_primitive_large_offsets.json.gz --check
generated_primitive_large_offsets.stream
actual->get_schema(actual, actual_schema.get()) failed with errno 45
*
/Users/dewey/Desktop/rscratch/arrow-nanoarrow/extensions/nanoarrow_ipc/src/apps/integration_test_util.cc:163
[FAIL] generated_extension.stream --check generated_extension.stream
-> Column 'dict_exts' missing key 'OFFSET'
[FAIL] generated_extension.json.gz --check generated_extension.json.gz
-> Column 'dict_exts' missing key 'OFFSET'
[FAIL] generated_extension.json.gz --check generated_extension.stream
[SKIP] generated_nested_dictionary.json.gz
[PASS] generated_nested_large_offsets.stream --check
generated_nested_large_offsets.stream
[PASS] generated_nested_large_offsets.json.gz --check
generated_nested_large_offsets.json.gz
[PASS] generated_nested_large_offsets.json.gz --check
generated_nested_large_offsets.stream
[PASS] generated_primitive_no_batches.stream --check
generated_primitive_no_batches.stream
[PASS] generated_primitive_no_batches.json.gz --check
generated_primitive_no_batches.json.gz
[PASS] generated_primitive_no_batches.json.gz --check
generated_primitive_no_batches.stream
```
---
.../src/nanoarrow/nanoarrow_ipc_reader.c | 4 +
.../src/nanoarrow/nanoarrow_ipc_reader_test.cc | 4 +-
src/nanoarrow/nanoarrow_testing.hpp | 541 +++++++++++++++++----
src/nanoarrow/nanoarrow_testing_test.cc | 518 ++++++++++++++++----
4 files changed, 898 insertions(+), 169 deletions(-)
diff --git a/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader.c
b/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader.c
index 275c016..87a20e9 100644
--- a/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader.c
+++ b/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader.c
@@ -146,6 +146,10 @@ static ArrowErrorCode ArrowIpcInputStreamFileRead(struct
ArrowIpcInputStream* st
ArrowErrorCode ArrowIpcInputStreamInitFile(struct ArrowIpcInputStream* stream,
void* file_ptr, int
close_on_release) {
+ if (file_ptr == NULL) {
+ return EINVAL;
+ }
+
struct ArrowIpcInputStreamFilePrivate* private_data =
(struct ArrowIpcInputStreamFilePrivate*)ArrowMalloc(
sizeof(struct ArrowIpcInputStreamFilePrivate));
diff --git
a/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader_test.cc
b/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader_test.cc
index 3441e68..4627955 100644
--- a/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader_test.cc
+++ b/extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_reader_test.cc
@@ -100,13 +100,15 @@ TEST(NanoarrowIpcReader, InputStreamBuffer) {
}
TEST(NanoarrowIpcReader, InputStreamFile) {
+ struct ArrowIpcInputStream stream;
+ ASSERT_EQ(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), EINVAL);
+
uint8_t input_data[] = {0x01, 0x02, 0x03, 0x04, 0x05};
FILE* file_ptr = tmpfile();
ASSERT_NE(file_ptr, nullptr);
ASSERT_EQ(fwrite(input_data, 1, sizeof(input_data), file_ptr),
sizeof(input_data));
fseek(file_ptr, 0, SEEK_SET);
- struct ArrowIpcInputStream stream;
uint8_t output_data[] = {0xff, 0xff, 0xff, 0xff, 0xff};
int64_t size_read_bytes;
diff --git a/src/nanoarrow/nanoarrow_testing.hpp
b/src/nanoarrow/nanoarrow_testing.hpp
index 15580b6..c2502a1 100644
--- a/src/nanoarrow/nanoarrow_testing.hpp
+++ b/src/nanoarrow/nanoarrow_testing.hpp
@@ -46,6 +46,16 @@ namespace testing {
/// \brief Writer for the Arrow integration testing JSON format
class TestingJSONWriter {
public:
+ TestingJSONWriter() : float_precision_(-1) {}
+
+ /// \brief Set the floating point precision of the writer
+ ///
+ /// The floating point precision by default is -1, which uses the JSON
serializer
+ /// to encode the value in the output. When writing files specifically for
+ /// integration tests, floating point values should be rounded to 3 decimal
places to
+ /// avoid serialization issues.
+ void set_float_precision(int precision) { float_precision_ = precision; }
+
/// \brief Write an ArrowArrayStream as a data file JSON object to out
///
/// Creates output like `{"schema": {...}, "batches": [...], ...}`.
@@ -114,8 +124,10 @@ class TestingJSONWriter {
}
// Write metadata
- out << R"(, "metadata": )";
- NANOARROW_RETURN_NOT_OK(WriteMetadata(out, schema->metadata));
+ if (schema->metadata != nullptr) {
+ out << R"(, "metadata": )";
+ NANOARROW_RETURN_NOT_OK(WriteMetadata(out, schema->metadata));
+ }
out << "}";
return NANOARROW_OK;
@@ -135,7 +147,7 @@ class TestingJSONWriter {
out << R"("name": null)";
} else {
out << R"("name": )";
- NANOARROW_RETURN_NOT_OK(WriteString(out, ArrowCharView(field->name)));
+ WriteString(out, ArrowCharView(field->name));
}
// Write nullability
@@ -166,8 +178,10 @@ class TestingJSONWriter {
// TODO: Dictionary (currently fails at WriteType)
// Write metadata
- out << R"(, "metadata": )";
- NANOARROW_RETURN_NOT_OK(WriteMetadata(out, field->metadata));
+ if (field->metadata != nullptr) {
+ out << R"(, "metadata": )";
+ NANOARROW_RETURN_NOT_OK(WriteMetadata(out, field->metadata));
+ }
out << "}";
return NANOARROW_OK;
@@ -183,11 +197,38 @@ class TestingJSONWriter {
return NANOARROW_OK;
}
+ /// \brief Write the metadata portion of a field
+ ///
+ /// Creates output like `[{"key": "...", "value": "..."}, ...]`.
+ ArrowErrorCode WriteMetadata(std::ostream& out, const char* metadata) {
+ if (metadata == nullptr) {
+ out << "null";
+ return NANOARROW_OK;
+ }
+
+ ArrowMetadataReader reader;
+ NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, metadata));
+ if (reader.remaining_keys == 0) {
+ out << "[]";
+ return NANOARROW_OK;
+ }
+
+ out << "[";
+ NANOARROW_RETURN_NOT_OK(WriteMetadataItem(out, &reader));
+ while (reader.remaining_keys > 0) {
+ out << ", ";
+ NANOARROW_RETURN_NOT_OK(WriteMetadataItem(out, &reader));
+ }
+
+ out << "]";
+ return NANOARROW_OK;
+ }
+
/// \brief Write a "batch" to out
///
/// Creates output like `{"count": 123, "columns": [...]}`.
ArrowErrorCode WriteBatch(std::ostream& out, const ArrowSchema* schema,
- ArrowArrayView* value) {
+ const ArrowArrayView* value) {
// Make sure we have a struct
if (std::string(schema->format) != "+s") {
return EINVAL;
@@ -210,7 +251,7 @@ class TestingJSONWriter {
///
/// Creates output like `{"name": "col", "count": 123, "VALIDITY": [...],
...}`.
ArrowErrorCode WriteColumn(std::ostream& out, const ArrowSchema* field,
- ArrowArrayView* value) {
+ const ArrowArrayView* value) {
out << "{";
// Write schema->name (may be null)
@@ -218,7 +259,7 @@ class TestingJSONWriter {
out << R"("name": null)";
} else {
out << R"("name": )";
- NANOARROW_RETURN_NOT_OK(WriteString(out, ArrowCharView(field->name)));
+ WriteString(out, ArrowCharView(field->name));
}
// Write length
@@ -275,6 +316,7 @@ class TestingJSONWriter {
case NANOARROW_TYPE_LIST:
case NANOARROW_TYPE_LARGE_LIST:
case NANOARROW_TYPE_FIXED_SIZE_LIST:
+ case NANOARROW_TYPE_MAP:
case NANOARROW_TYPE_DENSE_UNION:
case NANOARROW_TYPE_SPARSE_UNION:
break;
@@ -303,6 +345,8 @@ class TestingJSONWriter {
}
private:
+ int float_precision_;
+
ArrowErrorCode WriteType(std::ostream& out, const ArrowSchemaView* field) {
ArrowType type;
if (field->extension_name.data != nullptr) {
@@ -403,38 +447,14 @@ class TestingJSONWriter {
return NANOARROW_OK;
}
- ArrowErrorCode WriteMetadata(std::ostream& out, const char* metadata) {
- if (metadata == nullptr) {
- out << "null";
- return NANOARROW_OK;
- }
-
- ArrowMetadataReader reader;
- NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, metadata));
- if (reader.remaining_keys == 0) {
- out << "[]";
- return NANOARROW_OK;
- }
-
- out << "[";
- NANOARROW_RETURN_NOT_OK(WriteMetadataItem(out, &reader));
- while (reader.remaining_keys > 0) {
- out << ", ";
- NANOARROW_RETURN_NOT_OK(WriteMetadataItem(out, &reader));
- }
-
- out << "]";
- return NANOARROW_OK;
- }
-
ArrowErrorCode WriteMetadataItem(std::ostream& out, ArrowMetadataReader*
reader) {
ArrowStringView key;
ArrowStringView value;
NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderRead(reader, &key, &value));
out << R"({"key": )";
- NANOARROW_RETURN_NOT_OK(WriteString(out, key));
+ WriteString(out, key);
out << R"(, "value": )";
- NANOARROW_RETURN_NOT_OK(WriteString(out, value));
+ WriteString(out, value);
out << "}";
return NANOARROW_OK;
}
@@ -492,7 +512,7 @@ class TestingJSONWriter {
return NANOARROW_OK;
}
- ArrowErrorCode WriteData(std::ostream& out, ArrowArrayView* value) {
+ ArrowErrorCode WriteData(std::ostream& out, const ArrowArrayView* value) {
if (value->length == 0) {
out << "[]";
return NANOARROW_OK;
@@ -509,58 +529,59 @@ class TestingJSONWriter {
case NANOARROW_TYPE_INT32:
case NANOARROW_TYPE_UINT32:
// Regular JSON integers (i.e., 123456)
- out << ArrowArrayViewGetIntUnsafe(value, 0);
+ WriteIntMaybeNull(out, value, 0);
for (int64_t i = 1; i < value->length; i++) {
- out << ", " << ArrowArrayViewGetIntUnsafe(value, i);
+ out << ", ";
+ WriteIntMaybeNull(out, value, i);
}
break;
case NANOARROW_TYPE_INT64:
// Quoted integers to avoid overflow (i.e., "123456")
- out << R"(")" << ArrowArrayViewGetIntUnsafe(value, 0) << R"(")";
+ WriteQuotedIntMaybeNull(out, value, 0);
for (int64_t i = 1; i < value->length; i++) {
- out << R"(, ")" << ArrowArrayViewGetIntUnsafe(value, i) << R"(")";
+ out << ", ";
+ WriteQuotedIntMaybeNull(out, value, i);
}
break;
case NANOARROW_TYPE_UINT64:
// Quoted integers to avoid overflow (i.e., "123456")
- out << R"(")" << ArrowArrayViewGetUIntUnsafe(value, 0) << R"(")";
+ WriteQuotedUIntMaybeNull(out, value, 0);
for (int64_t i = 1; i < value->length; i++) {
- out << R"(, ")" << ArrowArrayViewGetUIntUnsafe(value, i) << R"(")";
+ out << ", ";
+ WriteQuotedUIntMaybeNull(out, value, i);
}
break;
case NANOARROW_TYPE_FLOAT:
case NANOARROW_TYPE_DOUBLE: {
- // JSON number to 3 decimal places
+ // JSON number to float_precision_ decimal places
LocalizedStream local_stream_opt(out);
- local_stream_opt.SetFixed(3);
+ local_stream_opt.SetFixed(float_precision_);
- out << ArrowArrayViewGetDoubleUnsafe(value, 0);
+ WriteFloatMaybeNull(out, value, 0);
for (int64_t i = 1; i < value->length; i++) {
- out << ", " << ArrowArrayViewGetDoubleUnsafe(value, i);
+ out << ", ";
+ WriteFloatMaybeNull(out, value, i);
}
break;
}
case NANOARROW_TYPE_STRING:
case NANOARROW_TYPE_LARGE_STRING:
- NANOARROW_RETURN_NOT_OK(
- WriteString(out, ArrowArrayViewGetStringUnsafe(value, 0)));
+ WriteString(out, ArrowArrayViewGetStringUnsafe(value, 0));
for (int64_t i = 1; i < value->length; i++) {
out << ", ";
- NANOARROW_RETURN_NOT_OK(
- WriteString(out, ArrowArrayViewGetStringUnsafe(value, i)));
+ WriteString(out, ArrowArrayViewGetStringUnsafe(value, i));
}
break;
case NANOARROW_TYPE_BINARY:
case NANOARROW_TYPE_LARGE_BINARY:
case NANOARROW_TYPE_FIXED_SIZE_BINARY: {
- NANOARROW_RETURN_NOT_OK(WriteBytes(out,
ArrowArrayViewGetBytesUnsafe(value, 0)));
+ WriteBytesMaybeNull(out, value, 0);
for (int64_t i = 1; i < value->length; i++) {
out << ", ";
- NANOARROW_RETURN_NOT_OK(
- WriteBytes(out, ArrowArrayViewGetBytesUnsafe(value, i)));
+ WriteBytesMaybeNull(out, value, i);
}
break;
}
@@ -574,7 +595,61 @@ class TestingJSONWriter {
return NANOARROW_OK;
}
- ArrowErrorCode WriteString(std::ostream& out, ArrowStringView value) {
+ void WriteIntMaybeNull(std::ostream& out, const ArrowArrayView* view,
int64_t i) {
+ if (ArrowArrayViewIsNull(view, i)) {
+ out << 0;
+ } else {
+ out << ArrowArrayViewGetIntUnsafe(view, i);
+ }
+ }
+
+ void WriteQuotedIntMaybeNull(std::ostream& out, const ArrowArrayView* view,
int64_t i) {
+ if (ArrowArrayViewIsNull(view, i)) {
+ out << R"("0")";
+ } else {
+ out << R"(")" << ArrowArrayViewGetIntUnsafe(view, i) << R"(")";
+ }
+ }
+
+ void WriteQuotedUIntMaybeNull(std::ostream& out, const ArrowArrayView* view,
+ int64_t i) {
+ if (ArrowArrayViewIsNull(view, i)) {
+ out << R"("0")";
+ } else {
+ out << R"(")" << ArrowArrayViewGetUIntUnsafe(view, i) << R"(")";
+ }
+ }
+
+ void WriteFloatMaybeNull(std::ostream& out, const ArrowArrayView* view,
int64_t i) {
+ if (float_precision_ >= 0) {
+ if (ArrowArrayViewIsNull(view, i)) {
+ out << static_cast<double>(0);
+ } else {
+ out << ArrowArrayViewGetDoubleUnsafe(view, i);
+ }
+ } else {
+ if (ArrowArrayViewIsNull(view, i)) {
+ out << "0.0";
+ } else {
+ out << nlohmann::json(ArrowArrayViewGetDoubleUnsafe(view, i));
+ }
+ }
+ }
+
+ void WriteBytesMaybeNull(std::ostream& out, const ArrowArrayView* view,
int64_t i) {
+ ArrowBufferView item = ArrowArrayViewGetBytesUnsafe(view, i);
+ if (ArrowArrayViewIsNull(view, i)) {
+ out << R"(")";
+ for (int64_t i = 0; i < item.size_bytes; i++) {
+ out << "00";
+ }
+ out << R"(")";
+ } else {
+ WriteBytes(out, item);
+ }
+ }
+
+ void WriteString(std::ostream& out, ArrowStringView value) {
out << R"(")";
for (int64_t i = 0; i < value.size_bytes; i++) {
@@ -583,12 +658,8 @@ class TestingJSONWriter {
out << R"(\")";
} else if (c == '\\') {
out << R"(\\)";
- } else if (c < 0) {
- // Not supporting multibyte unicode yet
- return ENOTSUP;
- } else if (c < 20) {
- // Data in the arrow-testing repo has a lot of content that requires
escaping
- // in this way (\uXXXX).
+ } else if (c >= 0 && c < 32) {
+ // Control characters need to be escaped with a \uXXXX escape
uint16_t utf16_bytes = static_cast<uint16_t>(c);
char utf16_esc[7];
@@ -601,10 +672,9 @@ class TestingJSONWriter {
}
out << R"(")";
- return NANOARROW_OK;
}
- ArrowErrorCode WriteBytes(std::ostream& out, ArrowBufferView value) {
+ void WriteBytes(std::ostream& out, ArrowBufferView value) {
out << R"(")";
char hex[3];
hex[2] = '\0';
@@ -614,11 +684,10 @@ class TestingJSONWriter {
out << hex;
}
out << R"(")";
- return NANOARROW_OK;
}
ArrowErrorCode WriteChildren(std::ostream& out, const ArrowSchema* field,
- ArrowArrayView* value) {
+ const ArrowArrayView* value) {
if (field->n_children == 0) {
out << "[]";
return NANOARROW_OK;
@@ -764,13 +833,12 @@ class TestingJSONReader {
// ArrowArrayView to enable validation
nanoarrow::UniqueArrayView array_view;
- NANOARROW_RETURN_NOT_OK(ArrowArrayViewInitFromSchema(
- array_view.get(), const_cast<ArrowSchema*>(schema), error));
+ NANOARROW_RETURN_NOT_OK(
+ ArrowArrayViewInitFromSchema(array_view.get(), schema, error));
// ArrowArray to hold memory
nanoarrow::UniqueArray array;
- NANOARROW_RETURN_NOT_OK(
- ArrowArrayInitFromSchema(array.get(),
const_cast<ArrowSchema*>(schema), error));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromSchema(array.get(), schema,
error));
NANOARROW_RETURN_NOT_OK(SetArrayBatch(obj, array_view.get(),
array.get(), error));
ArrowArrayMove(array.get(), out);
@@ -793,13 +861,12 @@ class TestingJSONReader {
// ArrowArrayView to enable validation
nanoarrow::UniqueArrayView array_view;
- NANOARROW_RETURN_NOT_OK(ArrowArrayViewInitFromSchema(
- array_view.get(), const_cast<ArrowSchema*>(schema), error));
+ NANOARROW_RETURN_NOT_OK(
+ ArrowArrayViewInitFromSchema(array_view.get(), schema, error));
// ArrowArray to hold memory
nanoarrow::UniqueArray array;
- NANOARROW_RETURN_NOT_OK(
- ArrowArrayInitFromSchema(array.get(),
const_cast<ArrowSchema*>(schema), error));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromSchema(array.get(), schema,
error));
// Parse the JSON into the array
NANOARROW_RETURN_NOT_OK(SetArrayColumn(obj, array_view.get(),
array.get(), error));
@@ -819,8 +886,6 @@ class TestingJSONReader {
Check(value.is_object(), error, "Expected Schema to be a JSON
object"));
NANOARROW_RETURN_NOT_OK(
Check(value.contains("fields"), error, "Schema missing key 'fields'"));
- NANOARROW_RETURN_NOT_OK(
- Check(value.contains("metadata"), error, "Schema missing key
'metadata'"));
NANOARROW_RETURN_NOT_OK_WITH_ERROR(
ArrowSchemaInitFromType(schema, NANOARROW_TYPE_STRUCT), error);
@@ -834,7 +899,9 @@ class TestingJSONReader {
NANOARROW_RETURN_NOT_OK(SetField(schema->children[i], fields[i], error));
}
- NANOARROW_RETURN_NOT_OK(SetMetadata(schema, value["metadata"], error));
+ if (value.contains("metadata")) {
+ NANOARROW_RETURN_NOT_OK(SetMetadata(schema, value["metadata"], error));
+ }
// Validate!
ArrowSchemaView schema_view;
@@ -853,8 +920,6 @@ class TestingJSONReader {
Check(value.contains("type"), error, "Field missing key 'type'"));
NANOARROW_RETURN_NOT_OK(
Check(value.contains("children"), error, "Field missing key
'children'"));
- NANOARROW_RETURN_NOT_OK(
- Check(value.contains("metadata"), error, "Field missing key
'metadata'"));
ArrowSchemaInit(schema);
@@ -887,7 +952,9 @@ class TestingJSONReader {
NANOARROW_RETURN_NOT_OK(SetField(schema->children[i], children[i],
error));
}
- NANOARROW_RETURN_NOT_OK(SetMetadata(schema, value["metadata"], error));
+ if (value.contains("metadata")) {
+ NANOARROW_RETURN_NOT_OK(SetMetadata(schema, value["metadata"], error));
+ }
// Validate!
ArrowSchemaView schema_view;
@@ -1055,19 +1122,24 @@ class TestingJSONReader {
ArrowErrorCode SetTypeDecimal(ArrowSchema* schema, const json& value,
ArrowError* error) {
- NANOARROW_RETURN_NOT_OK(Check(value.contains("bitWidth"), error,
- "Type[name=='decimal'] missing key
'bitWidth'"));
NANOARROW_RETURN_NOT_OK(Check(value.contains("precision"), error,
"Type[name=='decimal'] missing key
'precision'"));
NANOARROW_RETURN_NOT_OK(Check(value.contains("scale"), error,
"Type[name=='decimal'] missing key
'scale'"));
- const auto& bitWidth = value["bitWidth"];
- NANOARROW_RETURN_NOT_OK(Check(bitWidth.is_number_integer(), error,
- "Type[name=='decimal'] bitWidth must be
integer"));
+ // Some test files omit bitWidth for decimal128
+ int bit_width_int;
+ if (value.contains("bitWidth")) {
+ const auto& bit_width = value["bitWidth"];
+ NANOARROW_RETURN_NOT_OK(Check(bit_width.is_number_integer(), error,
+ "Type[name=='decimal'] bitWidth must be
integer"));
+ bit_width_int = bit_width.get<int>();
+ } else {
+ bit_width_int = 128;
+ }
ArrowType type;
- switch (bitWidth.get<int>()) {
+ switch (bit_width_int) {
case 128:
type = NANOARROW_TYPE_DECIMAL128;
break;
@@ -1661,6 +1733,309 @@ class TestingJSONReader {
}
};
+/// \brief Integration testing comparison utility
+///
+/// Utility to compare ArrowSchema, ArrowArray, and ArrowArrayStream instances.
+/// This should only be used in the context of integration testing as the
+/// comparison logic is specific to the integration testing JSON files and
+/// specification. Notably:
+///
+/// - Map types are considered equal regardless of the child names "entries",
+/// "key", and "value".
+/// - Float32 and Float64 values are compared according to their JSON
serialization.
+class TestingJSONComparison {
+ private:
+ // Internal representation of a human-readable inequality
+ struct Difference {
+ std::string path;
+ std::string actual;
+ std::string expected;
+ };
+
+ public:
+ /// \brief Returns the number of differences found by the previous call
+ size_t num_differences() const { return differences_.size(); }
+
+ /// \brief Dump a human-readable summary of differences to out
+ void WriteDifferences(std::ostream& out) {
+ for (const auto& difference : differences_) {
+ out << "Path: " << difference.path << "\n";
+ out << "- " << difference.actual << "\n";
+ out << "+ " << difference.expected << "\n";
+ out << "\n";
+ }
+ }
+
+ /// \brief Clear any existing differences
+ void ClearDifferences() { differences_.clear(); }
+
+ /// \brief Compare a stream of record batches
+ ///
+ /// Compares actual against expected using the following strategy:
+ ///
+ /// - Compares schemas for equality, returning if differences were found
+ /// - Compares pairs of record batches, returning if one stream finished
+ /// before another.
+ ///
+ /// Returns NANOARROW_OK if the comparison ran without error. Callers must
+ /// query num_differences() to obtain the result of the comparison on
success.
+ ArrowErrorCode CompareArrayStream(ArrowArrayStream* actual,
ArrowArrayStream* expected,
+ ArrowError* error = nullptr) {
+ // Read both schemas
+ nanoarrow::UniqueSchema actual_schema;
+ nanoarrow::UniqueSchema expected_schema;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(actual->get_schema(actual,
actual_schema.get()),
+ error);
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
+ expected->get_schema(expected, expected_schema.get()), error);
+
+ // Compare them and return if they are not equal
+ NANOARROW_RETURN_NOT_OK(
+ CompareSchema(expected_schema.get(), actual_schema.get(), error,
"Schema"));
+ if (num_differences() > 0) {
+ return NANOARROW_OK;
+ }
+
+ // Keep a record of the schema to compare batches
+ NANOARROW_RETURN_NOT_OK(SetSchema(expected_schema.get(), error));
+
+ int64_t n_batches = -1;
+ nanoarrow::UniqueArray actual_array;
+ nanoarrow::UniqueArray expected_array;
+ do {
+ n_batches++;
+ std::string batch_label = std::string("Batch ") +
std::to_string(n_batches);
+
+ // Read a batch from each stream
+ actual_array.reset();
+ expected_array.reset();
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(actual->get_next(actual,
actual_array.get()),
+ error);
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
+ expected->get_next(expected, expected_array.get()), error);
+
+ // Check the finished/unfinished status of both streams
+ if (actual_array->release == nullptr && expected_array->release !=
nullptr) {
+ differences_.push_back({batch_label, "finished stream", "unfinished
stream"});
+ return NANOARROW_OK;
+ }
+
+ if (actual_array->release != nullptr && expected_array->release ==
nullptr) {
+ differences_.push_back({batch_label, "unfinished stream", "finished
stream"});
+ return NANOARROW_OK;
+ }
+
+ // If both streams are done, break
+ if (actual_array->release == nullptr) {
+ break;
+ }
+
+ // Compare this batch
+ NANOARROW_RETURN_NOT_OK(
+ CompareBatch(actual_array.get(), expected_array.get(), error,
batch_label));
+ } while (true);
+
+ return NANOARROW_OK;
+ }
+
+ /// \brief Compare a top-level ArrowSchema struct
+ ///
+ /// Returns NANOARROW_OK if the comparison ran without error. Callers must
+ /// query num_differences() to obtain the result of the comparison on
success.
+ ArrowErrorCode CompareSchema(const ArrowSchema* actual, const ArrowSchema*
expected,
+ ArrowError* error = nullptr,
+ const std::string& path = "") {
+ // Compare the top-level schema "manually" because (1) map type needs
special-cased
+ // comparison and (2) it's easier to read the output if differences are
separated
+ // by field.
+ ArrowSchemaView actual_view;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaViewInit(&actual_view,
actual, nullptr),
+ error);
+
+ ArrowSchemaView expected_view;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
+ ArrowSchemaViewInit(&expected_view, expected, nullptr), error);
+
+ if (actual_view.type != NANOARROW_TYPE_STRUCT ||
+ expected_view.type != NANOARROW_TYPE_STRUCT) {
+ ArrowErrorSet(error, "Top-level schema must be struct");
+ return EINVAL;
+ }
+
+ // (Purposefully ignore the name field at the top level)
+
+ // Compare flags
+ if (actual->flags != expected->flags) {
+ differences_.push_back({path,
+ std::string(".flags: ") +
std::to_string(actual->flags),
+ std::string(".flags: ") +
std::to_string(expected->flags)});
+ }
+
+ // Compare children
+ if (actual->n_children != expected->n_children) {
+ differences_.push_back(
+ {path, std::string(".n_children: ") +
std::to_string(actual->n_children),
+ std::string(".n_children: ") +
std::to_string(expected->n_children)});
+ } else {
+ for (int64_t i = 0; i < expected->n_children; i++) {
+ NANOARROW_RETURN_NOT_OK(CompareField(
+ actual->children[i], expected->children[i], error,
+ path + std::string(".children[") + std::to_string(i) +
std::string("]")));
+ }
+ }
+
+ // Compare metadata
+ std::stringstream ss;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(writer_.WriteMetadata(ss,
actual->metadata),
+ error);
+ std::string actual_metadata = ss.str();
+
+ ss.str("");
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(writer_.WriteMetadata(ss,
expected->metadata),
+ error);
+ std::string expected_metadata = ss.str();
+
+ if (actual_metadata != expected_metadata) {
+ differences_.push_back({path, std::string(".metadata: ") +
actual_metadata,
+ std::string(".metadata: ") + expected_metadata});
+ }
+
+ return NANOARROW_OK;
+ }
+
+ /// \brief Set the ArrowSchema to be used to for future calls to
CompareBatch().
+ ArrowErrorCode SetSchema(const ArrowSchema* schema, ArrowError* error =
nullptr) {
+ schema_.reset();
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaDeepCopy(schema,
schema_.get()), error);
+ actual_.reset();
+ expected_.reset();
+
+ NANOARROW_RETURN_NOT_OK(
+ ArrowArrayViewInitFromSchema(actual_.get(), schema_.get(), error));
+ NANOARROW_RETURN_NOT_OK(
+ ArrowArrayViewInitFromSchema(expected_.get(), schema_.get(), error));
+
+ if (actual_->storage_type != NANOARROW_TYPE_STRUCT) {
+ ArrowErrorSet(error, "Can't SetSchema() with non-struct");
+ return EINVAL;
+ }
+
+ return NANOARROW_OK;
+ }
+
+ /// \brief Compare a top-level ArrowArray struct
+ ///
+ /// Returns NANOARROW_OK if the comparison ran without error. Callers must
+ /// query num_differences() to obtain the result of the comparison on
success.
+ ArrowErrorCode CompareBatch(const ArrowArray* actual, const ArrowArray*
expected,
+ ArrowError* error = nullptr, const std::string&
path = "") {
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArray(expected_.get(), expected,
error));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArray(actual_.get(), actual,
error));
+
+ if (actual->offset != expected->offset) {
+ differences_.push_back({path, ".offset: " +
std::to_string(actual->offset),
+ ".offset: " + std::to_string(expected->offset)});
+ }
+
+ if (actual->length != expected->length) {
+ differences_.push_back({path, ".length: " +
std::to_string(actual->length),
+ ".length: " + std::to_string(expected->length)});
+ }
+
+ // ArrowArrayViewSetArray() ensured that number of children of both match
schema
+ for (int64_t i = 0; i < expected_->n_children; i++) {
+ NANOARROW_RETURN_NOT_OK(CompareColumn(
+ schema_->children[i], actual_->children[i], expected_->children[i],
error,
+ path + std::string(".children[") + std::to_string(i) + "]"));
+ }
+
+ return NANOARROW_OK;
+ }
+
+ private:
+ TestingJSONWriter writer_;
+ std::vector<Difference> differences_;
+ nanoarrow::UniqueSchema schema_;
+ nanoarrow::UniqueArrayView actual_;
+ nanoarrow::UniqueArrayView expected_;
+
+ ArrowErrorCode CompareField(ArrowSchema* actual, ArrowSchema* expected,
+ ArrowError* error, const std::string& path = "")
{
+ // Preprocess both fields such that map types have canonical names
+ nanoarrow::UniqueSchema actual_copy;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaDeepCopy(actual,
actual_copy.get()),
+ error);
+ nanoarrow::UniqueSchema expected_copy;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaDeepCopy(expected,
expected_copy.get()),
+ error);
+
+
NANOARROW_RETURN_NOT_OK_WITH_ERROR(ForceMapNamesCanonical(actual_copy.get()),
error);
+
NANOARROW_RETURN_NOT_OK_WITH_ERROR(ForceMapNamesCanonical(expected_copy.get()),
+ error);
+ return CompareFieldBase(actual_copy.get(), expected_copy.get(), error,
path);
+ }
+
+ ArrowErrorCode CompareFieldBase(ArrowSchema* actual, ArrowSchema* expected,
+ ArrowError* error, const std::string& path =
"") {
+ std::stringstream ss;
+
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(writer_.WriteField(ss, expected),
error);
+ std::string expected_json = ss.str();
+
+ ss.str("");
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(writer_.WriteField(ss, actual), error);
+ std::string actual_json = ss.str();
+
+ if (actual_json != expected_json) {
+ differences_.push_back({path, actual_json, expected_json});
+ }
+
+ return NANOARROW_OK;
+ }
+
+ ArrowErrorCode CompareColumn(ArrowSchema* schema, ArrowArrayView* actual,
+ ArrowArrayView* expected, ArrowError* error,
+ const std::string& path = "") {
+ std::stringstream ss;
+
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(writer_.WriteColumn(ss, schema,
expected), error);
+ std::string expected_json = ss.str();
+
+ ss.str("");
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(writer_.WriteColumn(ss, schema,
actual), error);
+ std::string actual_json = ss.str();
+
+ if (actual_json != expected_json) {
+ differences_.push_back({path, actual_json, expected_json});
+ }
+
+ return NANOARROW_OK;
+ }
+
+ ArrowErrorCode ForceMapNamesCanonical(ArrowSchema* schema) {
+ ArrowSchemaView view;
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaViewInit(&view, schema, nullptr));
+
+ if (view.type == NANOARROW_TYPE_MAP) {
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0],
"entries"));
+ NANOARROW_RETURN_NOT_OK(
+ ArrowSchemaSetName(schema->children[0]->children[0], "key"));
+ NANOARROW_RETURN_NOT_OK(
+ ArrowSchemaSetName(schema->children[0]->children[1], "value"));
+ }
+
+ for (int64_t i = 0; i < schema->n_children; i++) {
+ NANOARROW_RETURN_NOT_OK(ForceMapNamesCanonical(schema->children[i]));
+ }
+
+ if (schema->dictionary != nullptr) {
+ NANOARROW_RETURN_NOT_OK(ForceMapNamesCanonical(schema->dictionary));
+ }
+
+ return NANOARROW_OK;
+ }
+};
+
/// @}
} // namespace testing
diff --git a/src/nanoarrow/nanoarrow_testing_test.cc
b/src/nanoarrow/nanoarrow_testing_test.cc
index 2fcc62a..d47159f 100644
--- a/src/nanoarrow/nanoarrow_testing_test.cc
+++ b/src/nanoarrow/nanoarrow_testing_test.cc
@@ -23,44 +23,41 @@
#include "nanoarrow/nanoarrow_testing.hpp"
+using nanoarrow::testing::TestingJSONComparison;
using nanoarrow::testing::TestingJSONReader;
using nanoarrow::testing::TestingJSONWriter;
-ArrowErrorCode WriteBatchJSON(std::ostream& out, const ArrowSchema* schema,
- ArrowArrayView* array_view) {
- TestingJSONWriter writer;
+ArrowErrorCode WriteBatchJSON(std::ostream& out, TestingJSONWriter& writer,
+ const ArrowSchema* schema, ArrowArrayView*
array_view) {
return writer.WriteBatch(out, schema, array_view);
}
-ArrowErrorCode WriteColumnJSON(std::ostream& out, const ArrowSchema* schema,
- ArrowArrayView* array_view) {
- TestingJSONWriter writer;
+ArrowErrorCode WriteColumnJSON(std::ostream& out, TestingJSONWriter& writer,
+ const ArrowSchema* schema, ArrowArrayView*
array_view) {
return writer.WriteColumn(out, schema, array_view);
}
-ArrowErrorCode WriteSchemaJSON(std::ostream& out, const ArrowSchema* schema,
- ArrowArrayView* array_view) {
- TestingJSONWriter writer;
+ArrowErrorCode WriteSchemaJSON(std::ostream& out, TestingJSONWriter& writer,
+ const ArrowSchema* schema, ArrowArrayView*
array_view) {
return writer.WriteSchema(out, schema);
}
-ArrowErrorCode WriteFieldJSON(std::ostream& out, const ArrowSchema* schema,
- ArrowArrayView* array_view) {
- TestingJSONWriter writer;
+ArrowErrorCode WriteFieldJSON(std::ostream& out, TestingJSONWriter& writer,
+ const ArrowSchema* schema, ArrowArrayView*
array_view) {
return writer.WriteField(out, schema);
}
-ArrowErrorCode WriteTypeJSON(std::ostream& out, const ArrowSchema* schema,
- ArrowArrayView* array_view) {
- TestingJSONWriter writer;
+ArrowErrorCode WriteTypeJSON(std::ostream& out, TestingJSONWriter& writer,
+ const ArrowSchema* schema, ArrowArrayView*
array_view) {
return writer.WriteType(out, schema);
}
void TestWriteJSON(std::function<ArrowErrorCode(ArrowSchema*)> type_expr,
std::function<ArrowErrorCode(ArrowArray*)> append_expr,
- ArrowErrorCode (*test_expr)(std::ostream&, const
ArrowSchema*,
- ArrowArrayView*),
- const std::string& expected_json) {
+ ArrowErrorCode (*test_expr)(std::ostream&,
TestingJSONWriter&,
+ const ArrowSchema*,
ArrowArrayView*),
+ const std::string& expected_json,
+ void (*setup_writer)(TestingJSONWriter& writer) = nullptr) {
std::stringstream ss;
nanoarrow::UniqueSchema schema;
@@ -76,7 +73,12 @@ void
TestWriteJSON(std::function<ArrowErrorCode(ArrowSchema*)> type_expr,
NANOARROW_OK);
ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr),
NANOARROW_OK);
- ASSERT_EQ(test_expr(ss, schema.get(), array_view.get()), NANOARROW_OK);
+ TestingJSONWriter writer;
+ if (setup_writer != nullptr) {
+ setup_writer(writer);
+ }
+
+ ASSERT_EQ(test_expr(ss, writer, schema.get(), array_view.get()),
NANOARROW_OK);
EXPECT_EQ(ss.str(), expected_json);
}
@@ -140,13 +142,13 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestColumnInt64) {
return ArrowSchemaInitFromType(schema, NANOARROW_TYPE_INT64);
},
[](ArrowArray* array) {
- NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt(array, 0));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt(array, 0));
return NANOARROW_OK;
},
&WriteColumnJSON,
- R"({"name": null, "count": 3, "VALIDITY": [1, 1, 1], "DATA": ["0", "1",
"0"]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": ["0", "1",
"0"]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestColumnUInt64) {
@@ -155,27 +157,44 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestColumnUInt64) {
return ArrowSchemaInitFromType(schema, NANOARROW_TYPE_UINT64);
},
[](ArrowArray* array) {
- NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt(array, 0));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt(array, 0));
return NANOARROW_OK;
},
&WriteColumnJSON,
- R"({"name": null, "count": 3, "VALIDITY": [1, 1, 1], "DATA": ["0", "1",
"0"]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": ["0", "1",
"0"]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestColumnFloat) {
+ // Test with constrained precision
+ TestWriteJSON(
+ [](ArrowSchema* schema) {
+ return ArrowSchemaInitFromType(schema, NANOARROW_TYPE_FLOAT);
+ },
+ [](ArrowArray* array) {
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendDouble(array, 0.1234));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendDouble(array, 1.2345));
+ return NANOARROW_OK;
+ },
+ &WriteColumnJSON,
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": [0.000,
0.123, 1.235]})",
+ [](TestingJSONWriter& writer) { writer.set_float_precision(3); });
+
TestWriteJSON(
[](ArrowSchema* schema) {
return ArrowSchemaInitFromType(schema, NANOARROW_TYPE_FLOAT);
},
[](ArrowArray* array) {
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendDouble(array, 0.1234));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendDouble(array, 1.2345));
return NANOARROW_OK;
},
&WriteColumnJSON,
- R"({"name": null, "count": 2, "VALIDITY": [1, 1], "DATA": [0.123,
1.235]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": [0.0,
0.1234000027179718, 1.2345000505447388]})",
+ [](TestingJSONWriter& writer) { writer.set_float_precision(-1); });
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestColumnString) {
@@ -184,13 +203,14 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestColumnString) {
return ArrowSchemaInitFromType(schema, NANOARROW_TYPE_STRING);
},
[](ArrowArray* array) {
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendString(array,
ArrowCharView("abc")));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendString(array,
ArrowCharView("def")));
return NANOARROW_OK;
},
&WriteColumnJSON,
- R"({"name": null, "count": 2, "VALIDITY": [1, 1], )"
- R"("OFFSET": [0, 3, 6], "DATA": ["abc", "def"]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], )"
+ R"("OFFSET": [0, 0, 3, 6], "DATA": ["", "abc", "def"]})");
// Check a string that requires escaping of characters \ and "
TestWriteJSON(
@@ -225,13 +245,14 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestColumnLargeString) {
return ArrowSchemaInitFromType(schema, NANOARROW_TYPE_LARGE_STRING);
},
[](ArrowArray* array) {
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendString(array,
ArrowCharView("abc")));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendString(array,
ArrowCharView("def")));
return NANOARROW_OK;
},
&WriteColumnJSON,
- R"({"name": null, "count": 2, "VALIDITY": [1, 1], )"
- R"("OFFSET": ["0", "3", "6"], "DATA": ["abc", "def"]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], )"
+ R"("OFFSET": ["0", "0", "3", "6"], "DATA": ["", "abc", "def"]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestColumnBinary) {
@@ -245,13 +266,34 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestColumnBinary) {
value_view.data.as_uint8 = value;
value_view.size_bytes = sizeof(value);
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendString(array,
ArrowCharView("abc")));
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendBytes(array, value_view));
return NANOARROW_OK;
},
&WriteColumnJSON,
- R"({"name": null, "count": 2, "VALIDITY": [1, 1], )"
- R"("OFFSET": [0, 3, 6], "DATA": ["616263", "0001FF"]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], )"
+ R"("OFFSET": [0, 0, 3, 6], "DATA": ["", "616263", "0001FF"]})");
+}
+
+TEST(NanoarrowTestingTest, NanoarrowTestingTestColumnFixedSizeBinary) {
+ TestWriteJSON(
+ [](ArrowSchema* schema) {
+ ArrowSchemaInit(schema);
+ return ArrowSchemaSetTypeFixedSize(schema,
NANOARROW_TYPE_FIXED_SIZE_BINARY, 3);
+ },
+ [](ArrowArray* array) {
+ uint8_t value[] = {0x00, 0x01, 0xff};
+ ArrowBufferView value_view;
+ value_view.data.as_uint8 = value;
+ value_view.size_bytes = sizeof(value);
+
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array, 1));
+ NANOARROW_RETURN_NOT_OK(ArrowArrayAppendBytes(array, value_view));
+ return NANOARROW_OK;
+ },
+ &WriteColumnJSON,
+ R"({"name": null, "count": 2, "VALIDITY": [0, 1], "DATA": ["000000",
"0001FF"]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestColumnStruct) {
@@ -317,7 +359,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestSchema) {
return NANOARROW_OK;
},
[](ArrowArray* array) { return NANOARROW_OK; }, &WriteSchemaJSON,
- R"({"fields": [], "metadata": null})");
+ R"({"fields": []})");
// More than zero fields
TestWriteJSON(
@@ -332,9 +374,8 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestSchema) {
},
[](ArrowArray* array) { return NANOARROW_OK; }, &WriteSchemaJSON,
R"({"fields": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}, )"
- R"({"name": null, "nullable": true, "type": {"name": "utf8"},
"children": [], "metadata": null}], )"
- R"("metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []}, )"
+ R"({"name": null, "nullable": true, "type": {"name": "utf8"},
"children": []}]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldBasic) {
@@ -344,7 +385,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldBasic) {
return NANOARROW_OK;
},
[](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})");
TestWriteJSON(
[](ArrowSchema* schema) {
@@ -353,7 +394,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldBasic) {
return NANOARROW_OK;
},
[](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,
- R"({"name": null, "nullable": false, "type": {"name": "null"},
"children": [], "metadata": null})");
+ R"({"name": null, "nullable": false, "type": {"name": "null"},
"children": []})");
TestWriteJSON(
[](ArrowSchema* schema) {
@@ -362,10 +403,19 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestFieldBasic) {
return NANOARROW_OK;
},
[](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,
- R"({"name": "colname", "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})");
+ R"({"name": "colname", "nullable": true, "type": {"name": "null"},
"children": []})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMetadata) {
+ // Missing metadata
+ TestWriteJSON(
+ [](ArrowSchema* schema) {
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaInitFromType(schema,
NANOARROW_TYPE_NA));
+ return NANOARROW_OK;
+ },
+ [](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})");
+
// Non-null but zero-size metadata
TestWriteJSON(
[](ArrowSchema* schema) {
@@ -409,9 +459,8 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldNested)
{
},
[](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,
R"({"name": null, "nullable": true, "type": {"name": "struct"},
"children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}, )"
- R"({"name": null, "nullable": true, "type": {"name": "utf8"},
"children": [], "metadata": null}], )"
- R"("metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []}, )"
+ R"({"name": null, "nullable": true, "type": {"name": "utf8"},
"children": []}]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestTypePrimitive) {
@@ -642,14 +691,23 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadSchema) {
ASSERT_EQ(
reader.ReadSchema(
R"({"fields": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}], )"
- R"("metadata": null})",
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})"
+ R"(], "metadata": [{"key": "k1", "value": "v1"}]})",
schema.get()),
NANOARROW_OK);
EXPECT_STREQ(schema->format, "+s");
ASSERT_EQ(schema->n_children, 1);
EXPECT_STREQ(schema->children[0]->format, "n");
+ ArrowMetadataReader metadata_reader;
+ ASSERT_EQ(ArrowMetadataReaderInit(&metadata_reader, schema->metadata),
NANOARROW_OK);
+ ASSERT_EQ(metadata_reader.remaining_keys, 1);
+ ArrowStringView key;
+ ArrowStringView value;
+ ASSERT_EQ(ArrowMetadataReaderRead(&metadata_reader, &key, &value),
NANOARROW_OK);
+ ASSERT_EQ(std::string(key.data, key.size_bytes), "k1");
+ ASSERT_EQ(std::string(value.data, value.size_bytes), "v1");
+
// Check invalid JSON
EXPECT_EQ(reader.ReadSchema(R"({)", schema.get()), EINVAL);
@@ -663,7 +721,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadFieldBasic) {
ASSERT_EQ(
reader.ReadField(
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})",
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})",
schema.get()),
NANOARROW_OK);
EXPECT_STREQ(schema->format, "n");
@@ -676,7 +734,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadFieldBasic) {
schema.reset();
ASSERT_EQ(
reader.ReadField(
- R"({"name": null, "nullable": false, "type": {"name": "null"},
"children": [], "metadata": null})",
+ R"({"name": null, "nullable": false, "type": {"name": "null"},
"children": []})",
schema.get()),
NANOARROW_OK);
EXPECT_FALSE(schema->flags & ARROW_FLAG_NULLABLE);
@@ -685,7 +743,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadFieldBasic) {
schema.reset();
ASSERT_EQ(
reader.ReadField(
- R"({"name": "colname", "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})",
+ R"({"name": "colname", "nullable": true, "type": {"name": "null"},
"children": []})",
schema.get()),
NANOARROW_OK);
EXPECT_STREQ(schema->name, "colname");
@@ -699,7 +757,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadFieldBasic) {
// Check that field is validated
EXPECT_EQ(
reader.ReadField(
- R"({"name": null, "nullable": true, "type": {"name":
"fixedsizebinary", "byteWidth": -1}, "children": [], "metadata": null})",
+ R"({"name": null, "nullable": true, "type": {"name":
"fixedsizebinary", "byteWidth": -1}, "children": []})",
schema.get()),
EINVAL);
}
@@ -738,7 +796,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadFieldNested) {
ASSERT_EQ(
reader.ReadField(
R"({"name": null, "nullable": true, "type": {"name": "struct"},
"children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}], )"
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []}], )"
R"("metadata": null})",
schema.get()),
NANOARROW_OK);
@@ -754,9 +812,8 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestRoundtripDataFile) {
std::string data_file_json =
R"({"schema": {"fields": [)"
- R"({"name": "col1", "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}, )"
- R"({"name": "col2", "nullable": true, "type": {"name": "utf8"},
"children": [], "metadata": null}], )"
- R"("metadata": null})"
+ R"({"name": "col1", "nullable": true, "type": {"name": "null"},
"children": []}, )"
+ R"({"name": "col2", "nullable": true, "type": {"name": "utf8"},
"children": []}]})"
R"(, "batches": [)"
R"({"count": 1, "columns": [)"
R"({"name": "col1", "count": 1}, )"
@@ -779,8 +836,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestRoundtripDataFile) {
data_file_json_roundtrip.str("");
// Check with zero batches
- std::string data_file_json_empty =
- R"({"schema": {"fields": [], "metadata": null}, "batches": []})";
+ std::string data_file_json_empty = R"({"schema": {"fields": []}, "batches":
[]})";
ASSERT_EQ(reader.ReadDataFile(data_file_json_empty, stream.get(), &error),
NANOARROW_OK)
<< error.message;
ASSERT_EQ(writer.WriteDataFile(data_file_json_roundtrip, stream.get()),
NANOARROW_OK);
@@ -830,7 +886,7 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestReadColumnBasic) {
ASSERT_EQ(
reader.ReadField(
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})",
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})",
schema.get()),
NANOARROW_OK);
@@ -899,7 +955,7 @@ void TestTypeRoundtrip(const std::string& type_json,
const std::string& column_json = "") {
std::stringstream field_json_builder;
field_json_builder << R"({"name": null, "nullable": true, "type": )" <<
type_json
- << R"(, "children": [], "metadata": null})";
+ << R"(, "children": []})";
TestFieldRoundtrip(field_json_builder.str(), column_json);
}
@@ -918,7 +974,7 @@ void TestTypeError(const std::string& type_json, const
std::string& msg,
int code = EINVAL) {
std::stringstream field_json_builder;
field_json_builder << R"({"name": null, "nullable": true, "type": )" <<
type_json
- << R"(, "children": [], "metadata": null})";
+ << R"(, "children": []})";
TestFieldError(field_json_builder.str(), msg, code);
}
@@ -990,10 +1046,10 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestFieldFloatingPoint) {
TestTypeRoundtrip(R"({"name": "floatingpoint", "precision": "HALF"})");
TestTypeRoundtrip(
R"({"name": "floatingpoint", "precision": "SINGLE"})",
- R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": [0.000,
1.230, 4.560]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": [0.0, 1.0,
2.0]})");
TestTypeRoundtrip(
R"({"name": "floatingpoint", "precision": "DOUBLE"})",
- R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": [0.000,
1.230, 4.560]})");
+ R"({"name": null, "count": 3, "VALIDITY": [0, 1, 1], "DATA": [0.0, 4.0,
5.0]})");
TestTypeError(
R"({"name": "floatingpoint", "precision": "NOT_A_PRECISION"})",
@@ -1014,6 +1070,16 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestFieldDecimal) {
TestTypeError(R"({"name": "decimal", "bitWidth": 123, "precision": 10,
"scale": 3})",
"Type[name=='decimal'] bitWidth must be 128 or 256");
+
+ // Ensure that omitted bitWidth maps to decimal128
+ TestingJSONReader reader;
+ nanoarrow::UniqueSchema schema;
+ ASSERT_EQ(
+ reader.ReadField(
+ R"({"name": null, "nullable": true, "type": {"name": "decimal",
"precision": 10, "scale": 3}, "children": []})",
+ schema.get()),
+ NANOARROW_OK);
+ EXPECT_STREQ(schema->format, "d:10,3");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMap) {
@@ -1021,84 +1087,84 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestFieldMap) {
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "map",
"keysSorted": true}, "children": [)"
R"({"name": "entries", "nullable": false, "type": {"name": "struct"},
"children": [)"
- R"({"name": null, "nullable": false, "type": {"name": "utf8"},
"children": [], "metadata": null}, )"
- R"({"name": null, "nullable": true, "type": {"name": "bool"},
"children": [], "metadata": null})"
- R"(], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": false, "type": {"name": "utf8"},
"children": []}, )"
+ R"({"name": null, "nullable": true, "type": {"name": "bool"},
"children": []})"
+ R"(]})"
+ R"(]})");
// Unsorted keys
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "map",
"keysSorted": false}, "children": [)"
R"({"name": "entries", "nullable": false, "type": {"name": "struct"},
"children": [)"
- R"({"name": null, "nullable": false, "type": {"name": "utf8"},
"children": [], "metadata": null}, )"
- R"({"name": null, "nullable": true, "type": {"name": "bool"},
"children": [], "metadata": null})"
- R"(], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": false, "type": {"name": "utf8"},
"children": []}, )"
+ R"({"name": null, "nullable": true, "type": {"name": "bool"},
"children": []})"
+ R"(]})"
+ R"(]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldStruct) {
// Empty
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "struct"},
"children": [)"
- R"(], "metadata": null})",
+ R"(]})",
R"({"name": null, "count": 0, "VALIDITY": [], "children": []})");
// Non-empty
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "struct"},
"children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})"
+ R"(]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldList) {
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "list"},
"children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})"
+ R"(]})");
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "largelist"},
"children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})"
+ R"(]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldFixedSizeList) {
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "fixedsizelist",
"listSize": 12}, "children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []})"
+ R"(]})");
}
TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldUnion) {
// Empty unions
TestFieldRoundtrip(
- R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"DENSE", "typeIds": []}, "children": [], "metadata": null})",
+ R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"DENSE", "typeIds": []}, "children": []})",
R"({"name": null, "count": 0, "TYPE_ID": [], "OFFSET": [], "children":
[]})");
TestFieldRoundtrip(
- R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"SPARSE", "typeIds": []}, "children": [], "metadata": null})",
+ R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"SPARSE", "typeIds": []}, "children": []})",
R"({"name": null, "count": 0, "TYPE_ID": [], "children": []})");
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"DENSE", "typeIds": [10,20]}, "children": [)"
- R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}, )"
- R"({"name": null, "nullable": true, "type": {"name": "utf8"},
"children": [], "metadata": null})"
- R"(], "metadata": null})");
+ R"({"name": null, "nullable": true, "type": {"name": "null"},
"children": []}, )"
+ R"({"name": null, "nullable": true, "type": {"name": "utf8"},
"children": []})"
+ R"(]})");
// Non-empty unions (null, "abc")
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"SPARSE", "typeIds": [10,20]}, "children": [)"
- R"({"name": "nulls", "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}, )"
- R"({"name": "strings", "nullable": true, "type": {"name": "utf8"},
"children": [], "metadata": null})"
- R"(], "metadata": null})",
+ R"({"name": "nulls", "nullable": true, "type": {"name": "null"},
"children": []}, )"
+ R"({"name": "strings", "nullable": true, "type": {"name": "utf8"},
"children": []})"
+ R"(]})",
R"({"name": null, "count": 2, "TYPE_ID": [20, 10], "children": [)"
R"({"name": "nulls", "count": 2}, )"
R"({"name": "strings", "count": 2, "VALIDITY": [1, 1], "OFFSET": [0, 3,
3], "DATA": ["abc", ""]})"
R"(]})");
TestFieldRoundtrip(
R"({"name": null, "nullable": true, "type": {"name": "union", "mode":
"DENSE", "typeIds": [10,20]}, "children": [)"
- R"({"name": "nulls", "nullable": true, "type": {"name": "null"},
"children": [], "metadata": null}, )"
- R"({"name": "strings", "nullable": true, "type": {"name": "utf8"},
"children": [], "metadata": null})"
- R"(], "metadata": null})",
+ R"({"name": "nulls", "nullable": true, "type": {"name": "null"},
"children": []}, )"
+ R"({"name": "strings", "nullable": true, "type": {"name": "utf8"},
"children": []})"
+ R"(]})",
R"({"name": null, "count": 2, "TYPE_ID": [20, 10], "OFFSET": [0, 0],
"children": [)"
R"({"name": "nulls", "count": 1}, )"
R"({"name": "strings", "count": 1, "VALIDITY": [1], "OFFSET": [0, 3],
"DATA": ["abc"]})"
@@ -1107,3 +1173,285 @@ TEST(NanoarrowTestingTest,
NanoarrowTestingTestFieldUnion) {
TestTypeError(R"({"name": "union", "mode": "NOT_A_MODE", "typeIds": []})",
"Type[name=='union'] mode must be 'DENSE' or 'SPARSE'");
}
+
+void AssertSchemasCompareEqual(ArrowSchema* actual, ArrowSchema* expected) {
+ TestingJSONComparison comparison;
+ std::stringstream msg;
+
+ ASSERT_EQ(comparison.CompareSchema(actual, expected), NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 0);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), "");
+}
+
+void AssertSchemasCompareUnequal(ArrowSchema* actual, ArrowSchema* expected,
+ int num_differences, const std::string&
differences) {
+ TestingJSONComparison comparison;
+ std::stringstream msg;
+
+ ASSERT_EQ(comparison.CompareSchema(actual, expected), NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), num_differences);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), differences);
+}
+
+TEST(NanoarrowTestingTest, NanoarrowTestingTestSchemaComparison) {
+ nanoarrow::UniqueSchema actual;
+ nanoarrow::UniqueSchema expected;
+
+ // Start with two identical schemas and ensure there are no differences
+ ArrowSchemaInit(actual.get());
+ ASSERT_EQ(ArrowSchemaSetTypeStruct(actual.get(), 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(actual->children[0], NANOARROW_TYPE_NA),
NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaDeepCopy(actual.get(), expected.get()), NANOARROW_OK);
+
+ AssertSchemasCompareEqual(actual.get(), expected.get());
+
+ // With different top-level flags
+ actual->flags = ARROW_FLAG_MAP_KEYS_SORTED;
+ AssertSchemasCompareUnequal(actual.get(), expected.get(),
/*num_differences*/ 1,
+ "Path: \n- .flags: 4\n+ .flags: 2\n\n");
+ actual->flags = expected->flags;
+
+ // With different top-level metadata
+ nanoarrow::UniqueBuffer buf;
+ ASSERT_EQ(ArrowMetadataBuilderInit(buf.get(), nullptr), NANOARROW_OK);
+ ASSERT_EQ(
+ ArrowMetadataBuilderAppend(buf.get(), ArrowCharView("key"),
ArrowCharView("value")),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetMetadata(actual.get(),
reinterpret_cast<char*>(buf->data)),
+ NANOARROW_OK);
+
+ AssertSchemasCompareUnequal(actual.get(), expected.get(),
/*num_differences*/ 1,
+ /*differences*/
+ "Path: "
+ R"(
+- .metadata: [{"key": "key", "value": "value"}]
++ .metadata: null
+
+)");
+ ASSERT_EQ(ArrowSchemaSetMetadata(actual.get(), nullptr), NANOARROW_OK);
+
+ // With different children
+ actual->children[0]->flags = 0;
+ AssertSchemasCompareUnequal(actual.get(), expected.get(),
/*num_differences*/ 1,
+ /*differences*/ R"(Path: .children[0]
+- {"name": null, "nullable": false, "type": {"name": "null"}, "children": []}
++ {"name": null, "nullable": true, "type": {"name": "null"}, "children": []}
+
+)");
+ actual->children[0]->flags = expected->children[0]->flags;
+
+ // With different numbers of children
+ actual.reset();
+ ArrowSchemaInit(actual.get());
+ ASSERT_EQ(ArrowSchemaSetTypeStruct(actual.get(), 0), NANOARROW_OK);
+ AssertSchemasCompareUnequal(
+ actual.get(), expected.get(), /*num_differences*/ 1,
+ /*differences*/ "Path: \n- .n_children: 0\n+ .n_children: 1\n\n");
+}
+
+TEST(NanoarrowTestingTest, NanoarrowTestingTestSchemaComparisonMap) {
+ nanoarrow::UniqueSchema actual;
+ nanoarrow::UniqueSchema expected;
+
+ // Start with two identical schemas with maps and ensure there are no
differences
+ ArrowSchemaInit(actual.get());
+ ASSERT_EQ(ArrowSchemaSetTypeStruct(actual.get(), 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(actual->children[0], NANOARROW_TYPE_MAP),
NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(actual->children[0]->children[0]->children[0],
+ NANOARROW_TYPE_STRING),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(actual->children[0]->children[0]->children[1],
+ NANOARROW_TYPE_INT32),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaDeepCopy(actual.get(), expected.get()), NANOARROW_OK);
+
+ AssertSchemasCompareEqual(actual.get(), expected.get());
+
+ // Even when one of the maps has different namees, there should be no
differences
+ ASSERT_EQ(
+ ArrowSchemaSetName(actual->children[0]->children[0], "this name is not
'entries'"),
+ NANOARROW_OK);
+ AssertSchemasCompareEqual(actual.get(), expected.get());
+
+ // This should also be true if the map is nested below the top-level of the
schema
+ nanoarrow::UniqueSchema actual2;
+ ASSERT_EQ(ArrowSchemaInitFromType(actual2.get(), NANOARROW_TYPE_STRUCT),
NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaAllocateChildren(actual2.get(), 1), NANOARROW_OK);
+ ArrowSchemaMove(actual.get(), actual2->children[0]);
+ expected.reset();
+ ASSERT_EQ(ArrowSchemaDeepCopy(actual2.get(), expected.get()), NANOARROW_OK);
+ ASSERT_EQ(
+ ArrowSchemaSetName(expected->children[0]->children[0]->children[0],
"entries"),
+ NANOARROW_OK);
+
+ AssertSchemasCompareEqual(actual2.get(), expected.get());
+}
+
+TEST(NanoarrowTestingTest, NanoarrowTestingTestArrayComparison) {
+ nanoarrow::UniqueSchema schema;
+ nanoarrow::UniqueArray actual;
+ nanoarrow::UniqueArray expected;
+ TestingJSONComparison comparison;
+ std::stringstream msg;
+
+ ArrowSchemaInit(schema.get());
+ ASSERT_EQ(ArrowSchemaSetTypeStruct(schema.get(), 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_NA),
NANOARROW_OK);
+ ASSERT_EQ(comparison.SetSchema(schema.get()), NANOARROW_OK);
+
+ ASSERT_EQ(ArrowArrayInitFromSchema(actual.get(), schema.get(), nullptr),
NANOARROW_OK);
+ ASSERT_EQ(ArrowArrayAppendNull(actual->children[0], 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowArrayFinishBuildingDefault(actual.get(), nullptr),
NANOARROW_OK);
+ ASSERT_EQ(ArrowArrayInitFromSchema(expected.get(), schema.get(), nullptr),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowArrayAppendNull(expected->children[0], 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowArrayFinishBuildingDefault(expected.get(), nullptr),
NANOARROW_OK);
+
+ ASSERT_EQ(comparison.CompareBatch(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 0);
+ comparison.ClearDifferences();
+
+ actual->length = 1;
+ ASSERT_EQ(comparison.CompareBatch(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), "Path: \n- .length: 1\n+ .length: 0\n\n");
+ msg.str("");
+ comparison.ClearDifferences();
+ actual->length = 0;
+
+ actual->offset = 1;
+ ASSERT_EQ(comparison.CompareBatch(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), "Path: \n- .offset: 1\n+ .offset: 0\n\n");
+ msg.str("");
+ comparison.ClearDifferences();
+ actual->offset = 0;
+
+ actual->children[0]->length = 2;
+ ASSERT_EQ(comparison.CompareBatch(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), R"(Path: .children[0]
+- {"name": null, "count": 2}
++ {"name": null, "count": 1}
+
+)");
+}
+
+ArrowErrorCode MakeArrayStream(const ArrowSchema* schema,
+ std::vector<std::string> batches_json,
+ ArrowArrayStream* out) {
+ TestingJSONReader reader;
+ nanoarrow::UniqueSchema schema_copy;
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaDeepCopy(schema, schema_copy.get()));
+ NANOARROW_RETURN_NOT_OK(
+ ArrowBasicArrayStreamInit(out, schema_copy.get(), batches_json.size()));
+
+ nanoarrow::UniqueArray array;
+ for (size_t i = 0; i < batches_json.size(); i++) {
+ NANOARROW_RETURN_NOT_OK(reader.ReadBatch(batches_json[i], schema,
array.get()));
+ ArrowBasicArrayStreamSetArray(out, i, array.get());
+ }
+
+ return NANOARROW_OK;
+}
+
+TEST(NanoarrowTestingTest, NanoarrowTestingTestArrayStreamComparison) {
+ nanoarrow::UniqueSchema schema;
+ nanoarrow::UniqueArrayStream actual;
+ nanoarrow::UniqueArrayStream expected;
+
+ std::string null1_batch_json =
+ R"({"count": 1, "columns": [{"name": null, "count": 1}]})";
+
+ TestingJSONComparison comparison;
+ std::stringstream msg;
+
+ ArrowSchemaInit(schema.get());
+ ASSERT_EQ(ArrowSchemaSetTypeStruct(schema.get(), 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_NA),
NANOARROW_OK);
+
+ // Identical streams with 0 batches
+ actual.reset();
+ expected.reset();
+ ASSERT_EQ(MakeArrayStream(schema.get(), {}, actual.get()), NANOARROW_OK);
+ ASSERT_EQ(MakeArrayStream(schema.get(), {}, expected.get()), NANOARROW_OK);
+ ASSERT_EQ(comparison.CompareArrayStream(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 0);
+ comparison.WriteDifferences(msg);
+
+ // Identical streams with >0 batches
+ actual.reset();
+ expected.reset();
+ ASSERT_EQ(MakeArrayStream(schema.get(), {null1_batch_json}, actual.get()),
+ NANOARROW_OK);
+ ASSERT_EQ(MakeArrayStream(schema.get(), {null1_batch_json}, expected.get()),
+ NANOARROW_OK);
+ ASSERT_EQ(comparison.CompareArrayStream(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 0);
+
+ // Stream where actual has more batches
+ actual.reset();
+ expected.reset();
+ ASSERT_EQ(MakeArrayStream(schema.get(), {null1_batch_json}, actual.get()),
+ NANOARROW_OK);
+ ASSERT_EQ(MakeArrayStream(schema.get(), {}, expected.get()), NANOARROW_OK);
+ ASSERT_EQ(comparison.CompareArrayStream(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), "Path: Batch 0\n- unfinished stream\n+ finished
stream\n\n");
+ msg.str("");
+ comparison.ClearDifferences();
+
+ // Stream where expected has more batches
+ actual.reset();
+ expected.reset();
+ ASSERT_EQ(MakeArrayStream(schema.get(), {}, actual.get()), NANOARROW_OK);
+ ASSERT_EQ(MakeArrayStream(schema.get(), {null1_batch_json}, expected.get()),
+ NANOARROW_OK);
+ ASSERT_EQ(comparison.CompareArrayStream(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), "Path: Batch 0\n- finished stream\n+ unfinished
stream\n\n");
+ msg.str("");
+ comparison.ClearDifferences();
+
+ // Stream where schemas differ
+ nanoarrow::UniqueSchema schema2;
+ ArrowSchemaInit(schema2.get());
+ ASSERT_EQ(ArrowSchemaSetTypeStruct(schema2.get(), 2), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(schema2->children[0], NANOARROW_TYPE_NA),
NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetType(schema2->children[1], NANOARROW_TYPE_NA),
NANOARROW_OK);
+ actual.reset();
+ expected.reset();
+ ASSERT_EQ(MakeArrayStream(schema2.get(), {}, actual.get()), NANOARROW_OK);
+ ASSERT_EQ(MakeArrayStream(schema.get(), {}, expected.get()), NANOARROW_OK);
+ ASSERT_EQ(comparison.CompareArrayStream(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), "Path: Schema\n- .n_children: 1\n+ .n_children: 2\n\n");
+ msg.str("");
+ comparison.ClearDifferences();
+
+ // Stream where batches differ
+ actual.reset();
+ expected.reset();
+ ASSERT_EQ(MakeArrayStream(schema.get(),
+ {R"({"count": 1, "columns": [{"name": null,
"count": 2}]})"},
+ actual.get()),
+ NANOARROW_OK);
+ ASSERT_EQ(MakeArrayStream(schema.get(), {null1_batch_json}, expected.get()),
+ NANOARROW_OK);
+ ASSERT_EQ(comparison.CompareArrayStream(actual.get(), expected.get()),
NANOARROW_OK);
+ EXPECT_EQ(comparison.num_differences(), 1);
+ comparison.WriteDifferences(msg);
+ EXPECT_EQ(msg.str(), R"(Path: Batch 0.children[0]
+- {"name": null, "count": 2}
++ {"name": null, "count": 1}
+
+)");
+}
