This is an automated email from the ASF dual-hosted git repository. CurtHagenlocher pushed a commit to branch ipc-message-custom-metadata in repository https://gitbox.apache.org/repos/asf/arrow-dotnet.git
commit 7a5bacf9b6df0a5d1588e38a1bd9e1418d4328b1 Author: Christoph Mettler <[email protected]> AuthorDate: Sun Mar 8 20:14:21 2026 +0100 Expose IPC Message custom_metadata on ArrowStreamReader The Arrow IPC format supports custom_metadata on each Message (RecordBatch), but the C# implementation currently ignores it on read. This adds a LastBatchCustomMetadata property to ArrowStreamReader that exposes the key-value pairs from the most recently read batch's Message. This is the read-side counterpart to pyarrow's read_next_batch_with_custom_metadata() and enables use cases like RPC frameworks that embed method routing or log metadata in per-batch custom_metadata fields. Co-Authored-By: Claude Opus 4.6 <[email protected]> --- src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs | 23 +++++++++++++++++++++++ src/Apache.Arrow/Ipc/ArrowStreamReader.cs | 8 ++++++++ 2 files changed, 31 insertions(+) diff --git a/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs b/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs index 2c380e5..acac52f 100644 --- a/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs +++ b/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs @@ -81,6 +81,11 @@ namespace Apache.Arrow.Ipc public abstract ValueTask<RecordBatch> ReadNextRecordBatchAsync(CancellationToken cancellationToken); public abstract RecordBatch ReadNextRecordBatch(); + /// <summary> + /// Custom metadata from the most recently read RecordBatch Message, if any. + /// </summary> + internal IReadOnlyDictionary<string, string> LastBatchCustomMetadata { get; private protected set; } + internal static T ReadMessage<T>(ByteBuffer bb) where T : struct, IFlatbufferObject { @@ -148,6 +153,7 @@ namespace Apache.Arrow.Ipc } List<IArrowArray> arrays = BuildArrays(message.Version, Schema, bodyByteBuffer, rb); + LastBatchCustomMetadata = ReadMessageCustomMetadata(message); return new RecordBatch(Schema, memoryOwner, arrays, (int)rb.Length); default: // NOTE: Skip unsupported message type @@ -158,6 +164,23 @@ namespace Apache.Arrow.Ipc return null; } + private static IReadOnlyDictionary<string, string> ReadMessageCustomMetadata(Flatbuf.Message message) + { + int count = message.CustomMetadataLength; + if (count == 0) + return null; + + var result = new Dictionary<string, string>(count); + for (int i = 0; i < count; i++) + { + Flatbuf.KeyValue kv = message.CustomMetadata(i).GetValueOrDefault(); + string key = kv.Key; + if (key != null) + result[key] = kv.Value ?? ""; + } + return result; + } + internal static ByteBuffer CreateByteBuffer(ReadOnlyMemory<byte> buffer) { return new ByteBuffer(new ReadOnlyMemoryBufferAllocator(buffer), 0); diff --git a/src/Apache.Arrow/Ipc/ArrowStreamReader.cs b/src/Apache.Arrow/Ipc/ArrowStreamReader.cs index afa3713..8379b12 100644 --- a/src/Apache.Arrow/Ipc/ArrowStreamReader.cs +++ b/src/Apache.Arrow/Ipc/ArrowStreamReader.cs @@ -14,6 +14,7 @@ // limitations under the License. using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -151,5 +152,12 @@ namespace Apache.Arrow.Ipc { return _implementation.ReadNextRecordBatch(); } + + /// <summary> + /// Custom metadata from the most recently read RecordBatch Message. + /// Updated after each call to ReadNextRecordBatch/ReadNextRecordBatchAsync. + /// Returns null if the last batch had no custom metadata. + /// </summary> + public IReadOnlyDictionary<string, string> LastBatchCustomMetadata => _implementation.LastBatchCustomMetadata; } }
