This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new cc67c6be feat: Allow the stream reader's validation level to be
configured (#964)
cc67c6be is described below
commit cc67c6becadd11d3890f36049fd2b41db3394fa7
Author: Rusty Conover <[email protected]>
AuthorDate: Sat Oct 17 00:33:13 2026 -0400
feat: Allow the stream reader's validation level to be configured (#964)
`ArrowIpcArrayStreamReader` always decodes with
`NANOARROW_VALIDATION_LEVEL_FULL`. That's the right default for bytes of
unknown origin. But full validation walks every offset of every string,
binary and list column and every dictionary index, and a caller that
already trusts its producer can't opt out of that pass. The lower-level
decoder functions already take a validation level; the stream reader
just doesn't expose one.
This adds a `validation_level` field to
`ArrowIpcArrayStreamReaderOptions`. It applies to both record batches
and dictionary batches. Passing `NULL` options keeps the current
behaviour (`FULL`). An out-of-range value makes
`ArrowIpcArrayStreamReaderInit()` return `EINVAL`.
Motivation: I'm moving a WebAssembly build of DuckDB from Arrow C++ to
nanoarrow. There, Arrow IPC streams written by Arrow JS in the same page
are read into the database. Arrow C++'s reader doesn't do full
validation by default, and the extra pass was a large part of an insert
slowdown I measured after switching.
Reading a 23 MB stream from an in-memory buffer (1M rows of int32,
double and string, in 16 batches), Release build, Apple M3, median of 50
runs:
| validation level | time |
|---|---|
| `FULL` (current behaviour) | 0.66 ms |
| `DEFAULT` | 0.38 ms |
The float64-only fixtures from `dev/benchmarks` don't change, because
full validation has nothing extra to check for them.
Compatibility: callers that fill in `ArrowIpcArrayStreamReaderOptions`
field by field need to set the new field as well. That's the same
situation as when `use_shared_buffers` was added in #165. The R and
Python bindings pass `NULL`, so they're unaffected. The `EINVAL` check
should catch most callers that miss the new field, rather than letting
them silently get a random level.
Tests: `StreamReaderValidationLevel` points a dictionary index past the
end of the dictionary in the existing dictionary fixture, then checks
that `FULL` rejects the batch and `DEFAULT` accepts it.
`StreamReaderInvalidValidationLevel` covers the `EINVAL` path. The two
existing tests that build options by hand now set the field. ctest
passes locally (macOS arm64, Debug, extra warnings). Under ASan/UBSan
the IPC tests pass, and the only failures are the same six allocation
tests that also fail on `main` under ASan.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
---
src/nanoarrow/ipc/reader.c | 23 +++++++++++---
src/nanoarrow/ipc/reader_test.cc | 69 ++++++++++++++++++++++++++++++++++++++++
src/nanoarrow/nanoarrow_ipc.h | 8 +++++
3 files changed, 96 insertions(+), 4 deletions(-)
diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c
index c699ca33..3348a9b6 100644
--- a/src/nanoarrow/ipc/reader.c
+++ b/src/nanoarrow/ipc/reader.c
@@ -187,6 +187,7 @@ struct ArrowIpcArrayStreamReaderPrivate {
struct ArrowIpcInputStream input;
struct ArrowIpcDecoder decoder;
int use_shared_buffers;
+ enum ArrowValidationLevel validation_level;
struct ArrowSchema out_schema;
int64_t field_index;
struct ArrowBuffer header;
@@ -459,7 +460,7 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch(
ArrowSharedBufferInit(&shared, &private_data->body),
&private_data->error);
ArrowErrorCode result =
ArrowIpcDecoderDecodeArrayFromSharedWithDictionaries(
&private_data->decoder, &shared, private_data->field_index,
- &private_data->dictionaries, out, NANOARROW_VALIDATION_LEVEL_FULL,
+ &private_data->dictionaries, out, private_data->validation_level,
&private_data->error);
ArrowBufferReset(&shared);
NANOARROW_RETURN_NOT_OK(result);
@@ -470,7 +471,7 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch(
NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayWithDictionaries(
&private_data->decoder, body_view, private_data->field_index,
- &private_data->dictionaries, out, NANOARROW_VALIDATION_LEVEL_FULL,
+ &private_data->dictionaries, out, private_data->validation_level,
&private_data->error));
}
@@ -488,7 +489,7 @@ static int ArrowIpcArrayStreamReaderProcessDictionary(
NANOARROW_RETURN_NOT_OK_WITH_ERROR(
ArrowSharedBufferInit(&shared, &private_data->body),
&private_data->error);
int result = ArrowIpcDecoderDecodeDictionaryFromShared(
- &private_data->decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL,
+ &private_data->decoder, &shared, private_data->validation_level,
&private_data->dictionaries, &private_data->error);
ArrowBufferReset(&shared);
NANOARROW_RETURN_NOT_OK(result);
@@ -497,7 +498,7 @@ static int ArrowIpcArrayStreamReaderProcessDictionary(
body_view.data.data = private_data->body.data;
body_view.size_bytes = private_data->body.size_bytes;
NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeDictionary(
- &private_data->decoder, body_view, NANOARROW_VALIDATION_LEVEL_FULL,
+ &private_data->decoder, body_view, private_data->validation_level,
&private_data->dictionaries, &private_data->error));
}
@@ -573,6 +574,18 @@ static const char* ArrowIpcArrayStreamReaderGetLastError(
ArrowErrorCode ArrowIpcArrayStreamReaderInit(
struct ArrowArrayStream* out, struct ArrowIpcInputStream* input_stream,
struct ArrowIpcArrayStreamReaderOptions* options) {
+ if (options != NULL) {
+ switch (options->validation_level) {
+ case NANOARROW_VALIDATION_LEVEL_NONE:
+ case NANOARROW_VALIDATION_LEVEL_MINIMAL:
+ case NANOARROW_VALIDATION_LEVEL_DEFAULT:
+ case NANOARROW_VALIDATION_LEVEL_FULL:
+ break;
+ default:
+ return EINVAL;
+ }
+ }
+
struct ArrowIpcArrayStreamReaderPrivate* private_data =
(struct ArrowIpcArrayStreamReaderPrivate*)ArrowMalloc(
sizeof(struct ArrowIpcArrayStreamReaderPrivate));
@@ -596,9 +609,11 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit(
if (options != NULL) {
private_data->field_index = options->field_index;
private_data->use_shared_buffers = options->use_shared_buffers;
+ private_data->validation_level = options->validation_level;
} else {
private_data->field_index = -1;
private_data->use_shared_buffers = ArrowSharedBufferIsThreadSafe();
+ private_data->validation_level = NANOARROW_VALIDATION_LEVEL_FULL;
}
out->private_data = private_data;
diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc
index d257e111..13545a69 100644
--- a/src/nanoarrow/ipc/reader_test.cc
+++ b/src/nanoarrow/ipc/reader_test.cc
@@ -19,6 +19,8 @@
#include <stdio.h>
+#include <string>
+
#include "nanoarrow/nanoarrow_ipc.h"
static uint8_t kSimpleSchema[] = {
@@ -272,6 +274,7 @@ TEST(NanoarrowIpcReader, StreamReaderBasicNoSharedBuffers) {
struct ArrowIpcArrayStreamReaderOptions options;
options.field_index = -1;
options.use_shared_buffers = 0;
+ options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL;
ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options),
NANOARROW_OK);
struct ArrowSchema schema;
@@ -495,6 +498,7 @@ TEST(NanoarrowIpcReader, StreamReaderUnsupportedFieldIndex)
{
struct ArrowIpcArrayStreamReaderOptions options;
options.field_index = 0;
options.use_shared_buffers = 0;
+ options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL;
ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options),
NANOARROW_OK);
struct ArrowSchema schema;
@@ -586,6 +590,71 @@ TEST(NanoarrowIpcReader, StreamReaderDictionary) {
ArrowArrayStreamRelease(&stream);
}
+TEST(NanoarrowIpcReader, StreamReaderValidationLevel) {
+ // Point the last dictionary index past the end of the three-value
dictionary.
+ // Only full validation inspects index values, so a lower level accepts the
batch.
+ uint8_t record_batch[sizeof(kDictionaryRecordBatch)];
+ memcpy(record_batch, kDictionaryRecordBatch, sizeof(kDictionaryRecordBatch));
+ ASSERT_EQ(record_batch[sizeof(record_batch) - 6], 0x02);
+ record_batch[sizeof(record_batch) - 6] = 0x07;
+
+ for (auto validation_level :
+ {NANOARROW_VALIDATION_LEVEL_FULL, NANOARROW_VALIDATION_LEVEL_DEFAULT}) {
+ SCOPED_TRACE(validation_level);
+ struct ArrowBuffer input_buffer;
+ ArrowBufferInit(&input_buffer);
+ ASSERT_EQ(
+ ArrowBufferAppend(&input_buffer, kDictionarySchema,
sizeof(kDictionarySchema)),
+ NANOARROW_OK);
+ ASSERT_EQ(
+ ArrowBufferAppend(&input_buffer, kDictionaryBatch,
sizeof(kDictionaryBatch)),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowBufferAppend(&input_buffer, record_batch,
sizeof(record_batch)),
+ NANOARROW_OK);
+
+ struct ArrowIpcInputStream input;
+ ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, &input_buffer),
NANOARROW_OK);
+
+ struct ArrowArrayStream stream;
+ struct ArrowIpcArrayStreamReaderOptions options;
+ options.field_index = -1;
+ options.use_shared_buffers = ArrowSharedBufferIsThreadSafe();
+ options.validation_level = validation_level;
+ ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options),
NANOARROW_OK);
+
+ struct ArrowArray array;
+ if (validation_level == NANOARROW_VALIDATION_LEVEL_FULL) {
+ ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr), EINVAL);
+ EXPECT_NE(std::string(ArrowArrayStreamGetLastError(&stream))
+ .find("Expected dictionary index >= 0 and < 3 but found
value 7"),
+ std::string::npos);
+ } else {
+ ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr),
NANOARROW_OK);
+ EXPECT_EQ(array.length, 3);
+ ArrowArrayRelease(&array);
+ }
+
+ ArrowArrayStreamRelease(&stream);
+ }
+}
+
+TEST(NanoarrowIpcReader, StreamReaderInvalidValidationLevel) {
+ struct ArrowBuffer input_buffer;
+ ArrowBufferInit(&input_buffer);
+ struct ArrowIpcInputStream input;
+ ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, &input_buffer),
NANOARROW_OK);
+
+ struct ArrowArrayStream stream;
+ struct ArrowIpcArrayStreamReaderOptions options;
+ options.field_index = -1;
+ options.use_shared_buffers = 0;
+ // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange):
deliberately invalid
+ options.validation_level = static_cast<ArrowValidationLevel>(4);
+ ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), EINVAL);
+
+ input.release(&input);
+}
+
TEST(NanoarrowIpcReader, StreamReaderDictionaryBatchWithoutDictionarySchema) {
// Send a dictionary batch when the schema has no dictionaries
struct ArrowBuffer input_buffer;
diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h
index 7f555975..38b57f97 100644
--- a/src/nanoarrow/nanoarrow_ipc.h
+++ b/src/nanoarrow/nanoarrow_ipc.h
@@ -936,6 +936,14 @@ struct ArrowIpcArrayStreamReaderOptions {
/// a single field there is probably no advantage to using shared buffers.
/// Defaults to the value of ArrowSharedBufferIsThreadSafe().
int use_shared_buffers;
+
+ /// \brief The level of validation to apply to each decoded array
+ ///
+ /// Defaults to NANOARROW_VALIDATION_LEVEL_FULL, which is appropriate for
untrusted
+ /// input. Use a lower level (e.g., NANOARROW_VALIDATION_LEVEL_DEFAULT) to
skip checks
+ /// that are O(n) in the length of the decoded buffers when the input comes
from a
+ /// trusted producer.
+ enum ArrowValidationLevel validation_level;
};
/// \brief Initialize an ArrowArrayStream from an input stream of bytes
