This is an automated email from the ASF dual-hosted git repository.
ianmcook pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new bb5d8d70 fix(ipc): Decode a DictionaryEncoding without indexType as
int32 indices (#957)
bb5d8d70 is described below
commit bb5d8d700b57fe99b8fdadf2babe680a3de60673
Author: Animesh Kumar <[email protected]>
AuthorDate: Fri Oct 9 08:47:23 2026 +0530
fix(ipc): Decode a DictionaryEncoding without indexType as int32 indices
(#957)
Closes #956
`Schema.fbs` says that when `DictionaryEncoding.indexType` is null, the
indices must be signed int32. The decoder passed the missing table
straight to `ArrowIpcDecoderSetTypeInt()`, which crashed (a segfault in
release builds, flatcc's "null pointer table access" assertion in debug
builds). It now uses int32 when the field isn't present.
The new `NanoarrowIpcDecodeDictionarySchemaWithoutIndexType` test
decodes a schema message with one `dictionary<values=string,
indices=int32>` field that pyarrow 25.0.1 wrote, with `indexType` then
removed from it. It crashes without the fix. I also read the
`WITHOUT_INDEX_TYPE` stream from the issue with
`ArrowIpcArrayStreamReader`: it now gives `struct<text:
dictionary(int32)<string>>` with the values `a b a`, the same as the
stream that has `indexType`.
`ctest` passes (384 tests, compression tests skipped), and pre-commit
passes on the changed files.
---
src/nanoarrow/ipc/decoder.c | 10 +++++++--
src/nanoarrow/ipc/decoder_test.cc | 44 +++++++++++++++++++++++++++++++++++++++
2 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c
index 9826b759..8eb91fe5 100644
--- a/src/nanoarrow/ipc/decoder.c
+++ b/src/nanoarrow/ipc/decoder.c
@@ -1246,8 +1246,14 @@ static int ArrowIpcSetDictionaryEncoding(
error);
NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaSetName(schema->dictionary,
""), error);
- NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt(
- schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)),
error));
+ // Schema.fbs: "If this field is null, the indices must be signed int32."
+ if (ns(DictionaryEncoding_indexType_is_present(dictionary_encoding))) {
+ NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt(
+ schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)),
error));
+ } else {
+ NANOARROW_RETURN_NOT_OK(
+ ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_INT32, error));
+ }
if (ns(DictionaryEncoding_isOrdered_get(dictionary_encoding))) {
schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED;
diff --git a/src/nanoarrow/ipc/decoder_test.cc
b/src/nanoarrow/ipc/decoder_test.cc
index dcdde176..b2cbaf7d 100644
--- a/src/nanoarrow/ipc/decoder_test.cc
+++ b/src/nanoarrow/ipc/decoder_test.cc
@@ -195,6 +195,23 @@ alignas(8) static uint8_t kDictionarySchema[] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00,
0x04, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+// Schema message with one field, some_col: dictionary<values=string,
indices=int32>,
+// written by pyarrow 25.0.1 with DictionaryEncoding.indexType removed (null
means
+// signed int32 per Schema.fbs)
+alignas(8) static uint8_t kDictionarySchemaNoIndexType[] = {
+ 0xff, 0xff, 0xff, 0xff, 0xa0, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x0a, 0x00, 0x0c, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00,
0x00, 0x00,
+ 0x00, 0x01, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb4, 0xff, 0xff, 0xff,
0x04, 0x00,
+ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00,
0x18, 0x00,
+ 0x08, 0x00, 0x06, 0x00, 0x07, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x14, 0x00,
0x10, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x01, 0x05, 0x14, 0x00, 0x00, 0x00, 0x48, 0x00,
0x00, 0x00,
+ 0x24, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x08, 0x00,
+ 0x00, 0x00, 0x73, 0x6f, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x00, 0x00,
0x00, 0x00,
+ 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0xd8, 0xff, 0xff, 0xff,
0x0c, 0x00,
+ 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x08, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
0x04, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00};
+
alignas(8) static uint8_t kDictionaryBatch[] = {
0xff, 0xff, 0xff, 0xff, 0xa8, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00,
0x00, 0x00,
0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00,
0x0c, 0x00,
@@ -664,6 +681,33 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema)
{
ArrowIpcDecoderReset(&decoder);
}
+TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchemaWithoutIndexType) {
+ struct ArrowIpcDecoder decoder;
+ struct ArrowError error;
+ struct ArrowSchema schema;
+
+ struct ArrowBufferView data;
+ data.data.as_uint8 = kDictionarySchemaNoIndexType;
+ data.size_bytes = sizeof(kDictionarySchemaNoIndexType);
+
+ ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK);
+
+ ASSERT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK)
+ << error.message;
+ ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA);
+
+ ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, &error),
NANOARROW_OK)
+ << error.message;
+ ASSERT_EQ(schema.n_children, 1);
+ EXPECT_STREQ(schema.children[0]->name, "some_col");
+ EXPECT_STREQ(schema.children[0]->format, "i");
+ ASSERT_NE(schema.children[0]->dictionary, nullptr);
+ EXPECT_STREQ(schema.children[0]->dictionary->format, "u");
+
+ ArrowSchemaRelease(&schema);
+ ArrowIpcDecoderReset(&decoder);
+}
+
TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryEncodingsUniqueIds) {
// Create dictionary encodings there is at least one duplicated identifier
struct ArrowIpcDictionaryEncodings dictionary_encodings;
