This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new ea215b00 fix(ipc): handle missing flatbuffer tables and dictionary
flags (#960)
ea215b00 is described below
commit ea215b000788ec2e94fc88bad5249290c94b01cd
Author: Dewey Dunnington <[email protected]>
AuthorDate: Mon Oct 12 21:12:04 2026 -0500
fix(ipc): handle missing flatbuffer tables and dictionary flags (#960)
Reject IPC messages with missing Field.type, Message.header, or
DictionaryBatch.data tables. Validate nested dictionary record-batch
node and buffer counts before accessing their vectors, returning EINVAL
for missing or mismatched vectors. Preserve outer-field nullability
while keeping dictionary values nullable.
Add regressions for malformed tables, missing/empty/incorrect vector
counts through both dictionary decoding APIs, recovery after rejected
batches, default int32 dictionary indices, and non-nullable dictionary
round trips.
Closes #958
Closes #959
Tests: 102 IPC-related CTests passed with ZSTD and LZ4 enabled;
pre-commit checks passed. The new vector-count regression reproduces the
assertion failure before the fix.
---
src/nanoarrow/ipc/decoder.c | 66 +++++++----
src/nanoarrow/ipc/decoder_test.cc | 235 ++++++++++++++++++++++++++++++++++----
src/nanoarrow/ipc/writer_test.cc | 3 +
3 files changed, 263 insertions(+), 41 deletions(-)
diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c
index 8eb91fe5..8953cb29 100644
--- a/src/nanoarrow/ipc/decoder.c
+++ b/src/nanoarrow/ipc/decoder.c
@@ -1139,26 +1139,30 @@ static int ArrowIpcDecoderSetTypeUnion(struct
ArrowSchema* schema,
static int ArrowIpcDecoderSetType(struct ArrowSchema* schema,
ns(Field_table_t) field,
int64_t n_children, struct ArrowError*
error) {
- int type_type = ns(Field_type_type(field));
+ ns(Type_union_type_t) type_type = ns(Field_type_type(field));
+ flatbuffers_generic_t type = ns(Field_type_get(field));
+ if (type == NULL) {
+ ArrowErrorSet(error, "Field type '%s' has no value",
ns(Type_type_name(type_type)));
+ return EINVAL;
+ }
+
switch (type_type) {
case ns(Type_Null):
return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_NA, error);
case ns(Type_Bool):
return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_BOOL, error);
case ns(Type_Int):
- return ArrowIpcDecoderSetTypeInt(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeInt(schema, type, error);
case ns(Type_FloatingPoint):
- return ArrowIpcDecoderSetTypeFloatingPoint(schema,
ns(Field_type_get(field)),
- error);
+ return ArrowIpcDecoderSetTypeFloatingPoint(schema, type, error);
case ns(Type_Decimal):
- return ArrowIpcDecoderSetTypeDecimal(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeDecimal(schema, type, error);
case ns(Type_Binary):
return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_BINARY,
error);
case ns(Type_LargeBinary):
return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_LARGE_BINARY,
error);
case ns(Type_FixedSizeBinary):
- return ArrowIpcDecoderSetTypeFixedSizeBinary(schema,
ns(Field_type_get(field)),
- error);
+ return ArrowIpcDecoderSetTypeFixedSizeBinary(schema, type, error);
case ns(Type_BinaryView):
ArrowErrorSet(error, "BinaryView not yet supported in IPC reader");
return ENOTSUP;
@@ -1170,15 +1174,15 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema*
schema, ns(Field_table_t)
ArrowErrorSet(error, "Utf8View not yet supported in IPC reader");
return ENOTSUP;
case ns(Type_Date):
- return ArrowIpcDecoderSetTypeDate(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeDate(schema, type, error);
case ns(Type_Time):
- return ArrowIpcDecoderSetTypeTime(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeTime(schema, type, error);
case ns(Type_Timestamp):
- return ArrowIpcDecoderSetTypeTimestamp(schema,
ns(Field_type_get(field)), error);
+ return ArrowIpcDecoderSetTypeTimestamp(schema, type, error);
case ns(Type_Duration):
- return ArrowIpcDecoderSetTypeDuration(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeDuration(schema, type, error);
case ns(Type_Interval):
- return ArrowIpcDecoderSetTypeInterval(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeInterval(schema, type, error);
case ns(Type_Struct_):
return ArrowIpcDecoderSetTypeSimpleNested(schema, "+s", error);
case ns(Type_List):
@@ -1186,17 +1190,15 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema*
schema, ns(Field_table_t)
case ns(Type_LargeList):
return ArrowIpcDecoderSetTypeSimpleNested(schema, "+L", error);
case ns(Type_FixedSizeList):
- return ArrowIpcDecoderSetTypeFixedSizeList(schema,
ns(Field_type_get(field)),
- error);
+ return ArrowIpcDecoderSetTypeFixedSizeList(schema, type, error);
case ns(Type_ListView):
case ns(Type_LargeListView):
ArrowErrorSet(error, "ListView/LargeListView not yet supported in IPC
reader");
return ENOTSUP;
case ns(Type_Map):
- return ArrowIpcDecoderSetTypeMap(schema, ns(Field_type_get(field)),
error);
+ return ArrowIpcDecoderSetTypeMap(schema, type, error);
case ns(Type_Union):
- return ArrowIpcDecoderSetTypeUnion(schema, ns(Field_type_get(field)),
n_children,
- error);
+ return ArrowIpcDecoderSetTypeUnion(schema, type, n_children, error);
case ns(Type_RunEndEncoded):
ArrowErrorSet(error, "RunEndEncoded not yet supported in IPC reader");
return ENOTSUP;
@@ -1229,6 +1231,9 @@ static int ArrowIpcSetDictionaryEncoding(
return EINVAL;
}
+ int64_t field_flags = schema->flags;
+ schema->flags |= ARROW_FLAG_NULLABLE;
+
struct ArrowSchema tmp;
ArrowSchemaMove(schema, &tmp);
@@ -1259,6 +1264,10 @@ static int ArrowIpcSetDictionaryEncoding(
schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED;
}
+ if (!(field_flags & ARROW_FLAG_NULLABLE)) {
+ schema->flags &= ~ARROW_FLAG_NULLABLE;
+ }
+
// Sort out field metadata between the schema and the dictionary member
NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowIpcMoveDictionaryMetadataBackToField(schema),
error);
@@ -1424,6 +1433,11 @@ static int ArrowIpcDecoderDecodeDictionaryBatchHeader(
(struct ArrowIpcDecoderPrivate*)decoder->private_data;
ns(DictionaryBatch_table_t) dictionary =
(ns(DictionaryBatch_table_t))message_header;
+ if (!ns(DictionaryBatch_data_is_present(dictionary))) {
+ ArrowErrorSet(error, "DictionaryBatch has no data");
+ return EINVAL;
+ }
+
private_data->dictionary.id = ns(DictionaryBatch_id(dictionary));
private_data->dictionary.is_delta = ns(DictionaryBatch_isDelta(dictionary));
@@ -1710,7 +1724,8 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct
ArrowIpcDecoder* decoder,
// Read some basic information from the message
decoder->metadata_version = ns(Message_version(message));
- decoder->message_type = ns(Message_header_type(message));
+ ns(MessageHeader_union_type_t) message_type =
ns(Message_header_type(message));
+ decoder->message_type = message_type;
decoder->body_size_bytes = ns(Message_bodyLength(message));
switch (decoder->metadata_version) {
@@ -1730,7 +1745,13 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct
ArrowIpcDecoder* decoder,
}
flatbuffers_generic_t message_header = ns(Message_header_get(message));
- switch (decoder->message_type) {
+ if (message_header == NULL) {
+ ArrowErrorSet(error, "Message header '%s' has no value",
+ ns(MessageHeader_type_name(message_type)));
+ return EINVAL;
+ }
+
+ switch (message_type) {
case ns(MessageHeader_Schema):
NANOARROW_RETURN_NOT_OK(
ArrowIpcDecoderDecodeSchemaHeader(decoder, message_header, error));
@@ -2776,11 +2797,14 @@ static ArrowErrorCode
ArrowIpcDecoderDecodeDictionaryInternal(
struct ArrowIpcDecoderPrivate* dictionary_decoder_private_data =
(struct ArrowIpcDecoderPrivate*)dictionary->decoder.private_data;
dictionary->decoder.message_type = NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH;
- dictionary_decoder_private_data->last_message = record_batch;
+ dictionary->decoder.metadata_version = decoder->metadata_version;
// Transfer the endianness and compression settings so that buffers are
byte-swapped
// and decompressed if needed (the nested decoder uses a default
decompressor)
dictionary_decoder_private_data->endianness = private_data->endianness;
- dictionary->decoder.codec = decoder->codec;
+ // Validate the nested batch's node and buffer counts before accessing its
vectors.
+ NANOARROW_RETURN_NOT_OK(
+ ArrowIpcDecoderDecodeRecordBatchHeader(&dictionary->decoder,
record_batch, error));
+ dictionary_decoder_private_data->last_message = record_batch;
struct ArrowArrayView* array_view;
NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayViewInternal(
diff --git a/src/nanoarrow/ipc/decoder_test.cc
b/src/nanoarrow/ipc/decoder_test.cc
index b2cbaf7d..3fb2bf6e 100644
--- a/src/nanoarrow/ipc/decoder_test.cc
+++ b/src/nanoarrow/ipc/decoder_test.cc
@@ -685,6 +685,7 @@ TEST(NanoarrowIpcTest,
NanoarrowIpcDecodeDictionarySchemaWithoutIndexType) {
struct ArrowIpcDecoder decoder;
struct ArrowError error;
struct ArrowSchema schema;
+ struct ArrowIpcDictionaryEncodings dictionary_encodings;
struct ArrowBufferView data;
data.data.as_uint8 = kDictionarySchemaNoIndexType;
@@ -696,15 +697,19 @@ TEST(NanoarrowIpcTest,
NanoarrowIpcDecodeDictionarySchemaWithoutIndexType) {
<< error.message;
ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA);
- ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, &error),
NANOARROW_OK)
+ ASSERT_EQ(ArrowIpcDecoderDecodeSchemaWithDictionaries(&decoder, &schema,
+ &dictionary_encodings,
&error),
+ NANOARROW_OK)
<< error.message;
ASSERT_EQ(schema.n_children, 1);
EXPECT_STREQ(schema.children[0]->name, "some_col");
EXPECT_STREQ(schema.children[0]->format, "i");
ASSERT_NE(schema.children[0]->dictionary, nullptr);
EXPECT_STREQ(schema.children[0]->dictionary->format, "u");
+ ASSERT_NE(ArrowIpcDictionaryEncodingsFindById(&dictionary_encodings, 0),
nullptr);
ArrowSchemaRelease(&schema);
+ ArrowIpcDictionaryEncodingsReset(&dictionary_encodings);
ArrowIpcDecoderReset(&decoder);
}
@@ -1430,6 +1435,30 @@ void AssertArrayViewIdentical(const struct
ArrowArrayView* actual,
}
}
+static void FinishEncapsulatedMessage(flatcc_builder_t* builder,
+ struct ArrowBuffer* out) {
+ // Encapsulate: continuation, little endian header size, header, padding to
8 bytes
+ size_t size = flatcc_builder_get_buffer_size(builder);
+ int64_t padded_size = _ArrowRoundUpToMultipleOf8(static_cast<int64_t>(size));
+ ASSERT_EQ(
+ ArrowBufferReserve(out, 2 * static_cast<int64_t>(sizeof(int32_t)) +
padded_size),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowBufferAppendInt32(out, -1), NANOARROW_OK);
+#if defined(__BIG_ENDIAN__)
+ ASSERT_EQ(ArrowBufferAppendInt32(
+ out,
static_cast<int32_t>(bswap32(static_cast<uint32_t>(padded_size)))),
+ NANOARROW_OK);
+#else
+ ASSERT_EQ(ArrowBufferAppendInt32(out, static_cast<int32_t>(padded_size)),
NANOARROW_OK);
+#endif
+ ASSERT_NE(flatcc_builder_copy_buffer(builder, out->data + out->size_bytes,
size),
+ nullptr);
+ out->size_bytes += size;
+ while (out->size_bytes % 8 != 0) {
+ out->data[out->size_bytes++] = 0;
+ }
+}
+
// Builds an encapsulated Schema message whose Message.custom_metadata and
// Schema.custom_metadata each contain a KeyValue with no key. Both `key` and
`value`
// are optional fields of KeyValue in the IPC format, so this passes flatbuffer
@@ -1462,30 +1491,196 @@ static void MakeKeylessMetadataSchemaMessage(struct
ArrowBuffer* out) {
ASSERT_EQ(ns(Message_bodyLength_add(&builder, 0)), 0);
ASSERT_NE(ns(Message_end_as_root(&builder)), 0);
- // Encapsulate: continuation, little endian header size, header, padding to
8 bytes
- size_t size = flatcc_builder_get_buffer_size(&builder);
- int64_t padded_size = _ArrowRoundUpToMultipleOf8(static_cast<int64_t>(size));
- ASSERT_EQ(
- ArrowBufferReserve(out, 2 * static_cast<int64_t>(sizeof(int32_t)) +
padded_size),
- NANOARROW_OK);
- ASSERT_EQ(ArrowBufferAppendInt32(out, -1), NANOARROW_OK);
-#if defined(__BIG_ENDIAN__)
- ASSERT_EQ(ArrowBufferAppendInt32(
- out,
static_cast<int32_t>(bswap32(static_cast<uint32_t>(padded_size)))),
- NANOARROW_OK);
-#else
- ASSERT_EQ(ArrowBufferAppendInt32(out, static_cast<int32_t>(padded_size)),
NANOARROW_OK);
-#endif
- ASSERT_NE(flatcc_builder_copy_buffer(&builder, out->data + out->size_bytes,
size),
- nullptr);
- out->size_bytes += size;
- while (out->size_bytes % 8 != 0) {
- out->data[out->size_bytes++] = 0;
+ ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out));
+ flatcc_builder_clear(&builder);
+}
+
+static void MakeSchemaWithMissingFieldType(struct ArrowBuffer* out) {
+ flatcc_builder_t builder;
+ flatcc_builder_init(&builder);
+
+ ASSERT_EQ(ns(Message_start_as_root(&builder)), 0);
+ ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0);
+ ASSERT_EQ(ns(Message_header_Schema_start(&builder)), 0);
+ ASSERT_EQ(ns(Schema_endianness_add(&builder, ns(Endianness_Little))), 0);
+ ASSERT_EQ(ns(Schema_fields_start(&builder)), 0);
+ ASSERT_EQ(ns(Schema_fields_push_start(&builder)), 0);
+ ASSERT_EQ(ns(Field_type_add_type(&builder, ns(Type_Int))), 0);
+ ASSERT_NE(ns(Schema_fields_push_end(&builder)), nullptr);
+ ASSERT_EQ(ns(Schema_fields_end(&builder)), 0);
+ ASSERT_EQ(ns(Message_header_Schema_end(&builder)), 0);
+ ASSERT_NE(ns(Message_end_as_root(&builder)), 0);
+
+ ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out));
+ flatcc_builder_clear(&builder);
+}
+
+static void MakeMessageWithMissingHeader(struct ArrowBuffer* out) {
+ flatcc_builder_t builder;
+ flatcc_builder_init(&builder);
+
+ ASSERT_EQ(ns(Message_start_as_root(&builder)), 0);
+ ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0);
+ ASSERT_EQ(ns(Message_header_add_type(&builder, ns(MessageHeader_Schema))),
0);
+ ASSERT_NE(ns(Message_end_as_root(&builder)), 0);
+
+ ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out));
+ flatcc_builder_clear(&builder);
+}
+
+static void MakeDictionaryBatchWithoutData(struct ArrowBuffer* out) {
+ flatcc_builder_t builder;
+ flatcc_builder_init(&builder);
+
+ ASSERT_EQ(ns(Message_start_as_root(&builder)), 0);
+ ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0);
+ ASSERT_EQ(ns(Message_header_DictionaryBatch_start(&builder)), 0);
+ ASSERT_EQ(ns(DictionaryBatch_id_add(&builder, 0)), 0);
+ ASSERT_EQ(ns(Message_header_DictionaryBatch_end(&builder)), 0);
+ ASSERT_NE(ns(Message_end_as_root(&builder)), 0);
+
+ ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out));
+ flatcc_builder_clear(&builder);
+}
+
+TEST(NanoarrowIpcTest, NanoarrowIpcDecodeMissingOptionalTables) {
+ struct ArrowError error;
+ nanoarrow::ipc::UniqueDecoder decoder;
+ ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK);
+
+ nanoarrow::UniqueBuffer missing_field_type;
+
ASSERT_NO_FATAL_FAILURE(MakeSchemaWithMissingFieldType(missing_field_type.get()));
+ struct ArrowBufferView data;
+ data.data.as_uint8 = missing_field_type->data;
+ data.size_bytes = missing_field_type->size_bytes;
+ ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error),
NANOARROW_OK)
+ << error.message;
+ ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error),
NANOARROW_OK)
+ << error.message;
+ nanoarrow::UniqueSchema schema;
+ EXPECT_EQ(ArrowIpcDecoderDecodeSchema(decoder.get(), schema.get(), &error),
EINVAL);
+ EXPECT_STREQ(error.message, "Field type 'Int' has no value");
+
+ nanoarrow::UniqueBuffer missing_header;
+ ASSERT_NO_FATAL_FAILURE(MakeMessageWithMissingHeader(missing_header.get()));
+ data.data.as_uint8 = missing_header->data;
+ data.size_bytes = missing_header->size_bytes;
+ ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error),
NANOARROW_OK)
+ << error.message;
+ EXPECT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), EINVAL);
+ EXPECT_STREQ(error.message, "Message header 'Schema' has no value");
+
+ nanoarrow::UniqueBuffer missing_dictionary_data;
+
ASSERT_NO_FATAL_FAILURE(MakeDictionaryBatchWithoutData(missing_dictionary_data.get()));
+ data.data.as_uint8 = missing_dictionary_data->data;
+ data.size_bytes = missing_dictionary_data->size_bytes;
+ ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error),
NANOARROW_OK)
+ << error.message;
+ EXPECT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), EINVAL);
+ EXPECT_STREQ(error.message, "DictionaryBatch has no data");
+}
+
+static void MakeDictionaryBatchWithVectorCounts(int64_t n_nodes, int64_t
n_buffers,
+ struct ArrowBuffer* out) {
+ flatcc_builder_t builder;
+ flatcc_builder_init(&builder);
+
+ ASSERT_EQ(ns(Message_start_as_root(&builder)), 0);
+ ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0);
+ ASSERT_EQ(ns(Message_header_DictionaryBatch_start(&builder)), 0);
+ ASSERT_EQ(ns(DictionaryBatch_data_start(&builder)), 0);
+ // Negative counts omit the optional vectors entirely.
+ if (n_nodes >= 0) {
+ std::vector<struct ns(FieldNode)> nodes(n_nodes);
+ ASSERT_EQ(ns(RecordBatch_nodes_create(&builder, nodes.data(),
nodes.size())), 0);
+ }
+ if (n_buffers >= 0) {
+ std::vector<struct ns(Buffer)> buffers(n_buffers);
+ ASSERT_EQ(ns(RecordBatch_buffers_create(&builder, buffers.data(),
buffers.size())),
+ 0);
}
+ ASSERT_EQ(ns(DictionaryBatch_data_end(&builder)), 0);
+ ASSERT_EQ(ns(Message_header_DictionaryBatch_end(&builder)), 0);
+ ASSERT_NE(ns(Message_end_as_root(&builder)), 0);
+ ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out));
flatcc_builder_clear(&builder);
}
+TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchInvalidVectorCounts) {
+ struct ArrowError error;
+ nanoarrow::ipc::UniqueDecoder decoder;
+ ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK);
+ struct ArrowBufferView data;
+ data.data.as_uint8 = kDictionarySchema;
+ data.size_bytes = sizeof(kDictionarySchema);
+ ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error),
NANOARROW_OK);
+ ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error),
NANOARROW_OK);
+
+ nanoarrow::UniqueSchema schema;
+ nanoarrow::ipc::UniqueDictionaryEncodings encodings;
+ ASSERT_EQ(ArrowIpcDecoderDecodeSchemaWithDictionaries(decoder.get(),
schema.get(),
+ encodings.get(),
&error),
+ NANOARROW_OK);
+ nanoarrow::ipc::UniqueDictionaries dictionaries;
+ ASSERT_EQ(ArrowIpcDictionariesInit(dictionaries.get(), encodings.get(),
&error),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowIpcDecoderSetEndianness(decoder.get(),
NANOARROW_IPC_ENDIANNESS_LITTLE),
+ NANOARROW_OK);
+
+ const int64_t counts[][2] = {{-1, 3}, {0, 3}, {2, 3}, {1, -1},
+ {1, 0}, {1, 1}, {1, 2}, {1, 4}};
+ for (const auto& count : counts) {
+ SCOPED_TRACE(::testing::Message()
+ << "nodes=" << count[0] << ", buffers=" << count[1]);
+ nanoarrow::UniqueBuffer message;
+ ASSERT_NO_FATAL_FAILURE(
+ MakeDictionaryBatchWithVectorCounts(count[0], count[1],
message.get()));
+ data.data.as_uint8 = message->data;
+ data.size_bytes = message->size_bytes;
+ ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error),
NANOARROW_OK)
+ << error.message;
+ ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error),
NANOARROW_OK)
+ << error.message;
+
+ struct ArrowBufferView body = {};
+ std::string expected;
+ if (count[0] != 1) {
+ expected = "Expected 1 field nodes in message but found " +
+ std::to_string(count[0] < 0 ? 0 : count[0]);
+ } else {
+ expected = "Expected 3 buffers in message but found " +
+ std::to_string(count[1] < 0 ? 0 : count[1]);
+ }
+ EXPECT_EQ(ArrowIpcDecoderDecodeDictionary(decoder.get(), body,
+ NANOARROW_VALIDATION_LEVEL_FULL,
+ dictionaries.get(), &error),
+ EINVAL);
+ EXPECT_STREQ(error.message, expected.c_str());
+
+ nanoarrow::UniqueBuffer shared_body;
+ EXPECT_EQ(ArrowIpcDecoderDecodeDictionaryFromShared(decoder.get(),
shared_body.get(),
+
NANOARROW_VALIDATION_LEVEL_FULL,
+ dictionaries.get(),
&error),
+ EINVAL);
+ EXPECT_STREQ(error.message, expected.c_str());
+ }
+
+ // A rejected dictionary batch must not prevent decoding the next valid
batch.
+ data.data.as_uint8 = kDictionaryBatch;
+ data.size_bytes = sizeof(kDictionaryBatch);
+ ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error),
NANOARROW_OK);
+ ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error),
NANOARROW_OK);
+ struct ArrowBufferView body;
+ body.data.as_uint8 = data.data.as_uint8 + decoder->header_size_bytes;
+ body.size_bytes = decoder->body_size_bytes;
+ ASSERT_EQ(ArrowIpcDecoderDecodeDictionary(decoder.get(), body,
+ NANOARROW_VALIDATION_LEVEL_FULL,
+ dictionaries.get(), &error),
+ NANOARROW_OK)
+ << error.message;
+}
+
TEST(NanoarrowIpcTest, NanoarrowIpcDecodeMetadataWithoutKey) {
nanoarrow::UniqueBuffer message;
ASSERT_NO_FATAL_FAILURE(MakeKeylessMetadataSchemaMessage(message.get()));
diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc
index 3b08d0c8..edcc8839 100644
--- a/src/nanoarrow/ipc/writer_test.cc
+++ b/src/nanoarrow/ipc/writer_test.cc
@@ -339,6 +339,7 @@ TEST(NanoarrowIpcWriter, RoundtripDictionaryStream) {
nanoarrow::UniqueSchema schema;
nanoarrow::UniqueArray array;
MakeDictionaryStructArray(array.get(), schema.get());
+ schema->children[0]->flags &= ~ARROW_FLAG_NULLABLE;
nanoarrow::UniqueArrayStream array_stream;
ASSERT_EQ(ArrowBasicArrayStreamInit(array_stream.get(), schema.get(), 1),
NANOARROW_OK);
@@ -366,7 +367,9 @@ TEST(NanoarrowIpcWriter, RoundtripDictionaryStream) {
NANOARROW_OK)
<< error.message;
ASSERT_EQ(roundtrip_schema->n_children, 1);
+ EXPECT_EQ(roundtrip_schema->children[0]->flags & ARROW_FLAG_NULLABLE, 0);
ASSERT_NE(roundtrip_schema->children[0]->dictionary, nullptr);
+ EXPECT_NE(roundtrip_schema->children[0]->dictionary->flags &
ARROW_FLAG_NULLABLE, 0);
EXPECT_STREQ(roundtrip_schema->children[0]->dictionary->format, "u");
nanoarrow::UniqueArray roundtrip_array;
