Use stdatomic memory orders for reference counts, ticket locks, sequence counters and cache-aware IPC queues. Large payloads are represented by mapped out-of-line memory descriptors, while the message path copies only descriptors.
Signed-off-by: Breno Rodrigues Alves <[email protected]> --- modern/include/mach/c17/atomic.h | 245 ++++++++++++++ modern/include/mach/c17/ipc.h | 177 ++++++++++ modern/src/ipc/ipc.c | 542 +++++++++++++++++++++++++++++++ modern/tests/test_ipc.c | 189 +++++++++++ 4 files changed, 1153 insertions(+) create mode 100644 modern/include/mach/c17/atomic.h create mode 100644 modern/include/mach/c17/ipc.h create mode 100644 modern/src/ipc/ipc.c create mode 100644 modern/tests/test_ipc.c diff --git a/modern/include/mach/c17/atomic.h b/modern/include/mach/c17/atomic.h new file mode 100644 index 00000000..e87ee4f6 --- /dev/null +++ b/modern/include/mach/c17/atomic.h @@ -0,0 +1,245 @@ +/* Portable atomic and locking primitives for the C17 Mach kernel. + Copyright (C) 2026 Free Software Foundation, Inc. + + This file is part of GNU Mach. See base.h for the license. + + Everything here is expressed with ISO C17 <stdatomic.h>. The old + assembly locks (machine/lock.h, i386/lock.s) implemented exactly the + same state machine with a different spelling; keeping one spelling + lets the compiler choose the best instruction and, more importantly, + lets the memory model be reasoned about by humans. */ + +#ifndef _MACH_C17_ATOMIC_H +#define _MACH_C17_ATOMIC_H 1 + +#include <stdatomic.h> + +#include <mach/c17/base.h> + +/* --------------------------------------------------------------------- + Relaxed and ordering-explicit scalar accesses. */ + +typedef _Atomic uint32_t mach_atomic_u32; +typedef _Atomic uint64_t mach_atomic_u64; +typedef _Atomic size_t mach_atomic_size; +typedef _Atomic mach_phys_t mach_atomic_phys; +typedef _Atomic bool mach_atomic_bool; + +MACH_INLINE uint32_t +mach_atomic_load_u32 (const mach_atomic_u32 *object, memory_order order) +{ + return atomic_load_explicit (object, order); +} + +MACH_INLINE void +mach_atomic_store_u32 (mach_atomic_u32 *object, uint32_t value, + memory_order order) +{ + atomic_store_explicit (object, value, order); +} + +MACH_INLINE uint64_t +mach_atomic_load_u64 (const mach_atomic_u64 *object, memory_order order) +{ + return atomic_load_explicit (object, order); +} + +MACH_INLINE void +mach_atomic_store_u64 (mach_atomic_u64 *object, uint64_t value, + memory_order order) +{ + atomic_store_explicit (object, value, order); +} + +MACH_INLINE size_t +mach_atomic_load_size (const mach_atomic_size *object, memory_order order) +{ + return atomic_load_explicit (object, order); +} + +MACH_INLINE void +mach_atomic_store_size (mach_atomic_size *object, size_t value, + memory_order order) +{ + atomic_store_explicit (object, value, order); +} + +/* --------------------------------------------------------------------- + Reference counting. + + A reference count is incremented with relaxed ordering: the caller + must already hold a reference. Destruction uses acquire-release on + the final decrement so that all stores performed by the previous + owners are visible to the thread that frees the object. */ + +typedef struct mach_refcount +{ + mach_atomic_u32 count; +} mach_refcount_t; + +#define MACH_REFCOUNT_INIT(value) { (value) } + +MACH_INLINE void +mach_refcount_init (mach_refcount_t *refcount, uint32_t value) +{ + atomic_init (&refcount->count, value); +} + +MACH_INLINE uint32_t +mach_refcount_load (const mach_refcount_t *refcount) +{ + return atomic_load_explicit (&refcount->count, memory_order_relaxed); +} + +MACH_INLINE void +mach_refcount_inc (mach_refcount_t *refcount) +{ + atomic_fetch_add_explicit (&refcount->count, 1u, memory_order_relaxed); +} + +/* Increment unless the count is already zero. Used by IPC to revive an + object that is racing with its final release. */ +MACH_INLINE bool +mach_refcount_inc_not_zero (mach_refcount_t *refcount) +{ + uint32_t value; + + value = atomic_load_explicit (&refcount->count, memory_order_relaxed); + do + { + if (value == 0) + return false; + } + while (!atomic_compare_exchange_weak_explicit (&refcount->count, &value, + value + 1u, + memory_order_relaxed, + memory_order_relaxed)); + + return true; +} + +/* Return true when the caller dropped the last reference. */ +MACH_INLINE bool +mach_refcount_dec_and_test (mach_refcount_t *refcount) +{ + return atomic_fetch_sub_explicit (&refcount->count, 1u, + memory_order_acq_rel) == 1u; +} + +/* --------------------------------------------------------------------- + Spin locks. + + The ticket lock hands out a fair FIFO order and keeps the two hot + halves on separate cache lines. It is used only for very short + critical sections, never around anything that can fault or block. */ + +typedef struct MACH_CACHELINE_ALIGN mach_spinlock +{ + mach_atomic_u32 next_ticket MACH_CACHELINE_ALIGN; + mach_atomic_u32 now_serving MACH_CACHELINE_ALIGN; +} mach_spinlock_t; + +#define MACH_SPINLOCK_INIT { 0u, 0u } + +MACH_INLINE void +mach_spinlock_init (mach_spinlock_t *lock) +{ + atomic_init (&lock->next_ticket, 0u); + atomic_init (&lock->now_serving, 0u); +} + +MACH_INLINE void +mach_spinlock_lock (mach_spinlock_t *lock) +{ + uint32_t ticket; + + ticket = atomic_fetch_add_explicit (&lock->next_ticket, 1u, + memory_order_relaxed); + while (atomic_load_explicit (&lock->now_serving, + memory_order_acquire) != ticket) + mach_cpu_relax (); +} + +MACH_INLINE bool +mach_spinlock_try_lock (mach_spinlock_t *lock) +{ + uint32_t serving; + + serving = atomic_load_explicit (&lock->now_serving, memory_order_relaxed); + return atomic_compare_exchange_strong_explicit (&lock->next_ticket, + &serving, serving + 1u, + memory_order_acquire, + memory_order_relaxed); +} + +MACH_INLINE void +mach_spinlock_unlock (mach_spinlock_t *lock) +{ + atomic_fetch_add_explicit (&lock->now_serving, 1u, memory_order_release); +} + +MACH_INLINE bool +mach_spinlock_is_locked (const mach_spinlock_t *lock) +{ + return atomic_load_explicit (&lock->now_serving, memory_order_relaxed) + != atomic_load_explicit (&lock->next_ticket, memory_order_relaxed); +} + +/* --------------------------------------------------------------------- + Sequence counters. + + A seqlock lets readers retry instead of blocking and is a natural fit + for the scheduler's statistics and for the VM's free-page counters, + both of which are read far more often than written. */ + +typedef struct mach_seqlock +{ + mach_atomic_u32 sequence; +} mach_seqlock_t; + +#define MACH_SEQLOCK_INIT { 0u } + +MACH_INLINE void +mach_seqlock_init (mach_seqlock_t *seqlock) +{ + atomic_init (&seqlock->sequence, 0u); +} + +MACH_INLINE void +mach_seqlock_write_begin (mach_seqlock_t *seqlock) +{ + /* Odd values mark a write in progress, which also serialises the + writers among themselves. */ + while (atomic_fetch_add_explicit (&seqlock->sequence, 1u, + memory_order_acquire) & 1u) + mach_cpu_relax (); +} + +MACH_INLINE void +mach_seqlock_write_end (mach_seqlock_t *seqlock) +{ + atomic_fetch_add_explicit (&seqlock->sequence, 1u, memory_order_release); +} + +MACH_INLINE uint32_t +mach_seqlock_read_begin (const mach_seqlock_t *seqlock) +{ + uint32_t sequence; + + do + sequence = atomic_load_explicit (&seqlock->sequence, + memory_order_acquire); + while (sequence & 1u); + + return sequence; +} + +MACH_INLINE bool +mach_seqlock_read_retry (const mach_seqlock_t *seqlock, uint32_t sequence) +{ + atomic_thread_fence (memory_order_acquire); + return atomic_load_explicit (&seqlock->sequence, memory_order_relaxed) + != sequence; +} + +#endif /* _MACH_C17_ATOMIC_H */ diff --git a/modern/include/mach/c17/ipc.h b/modern/include/mach/c17/ipc.h new file mode 100644 index 00000000..35648e9e --- /dev/null +++ b/modern/include/mach/c17/ipc.h @@ -0,0 +1,177 @@ +/* High-performance Mach IPC: ports, cache-friendly queues, zero copy. + Copyright (C) 2026 Free Software Foundation, Inc. + + This file is part of GNU Mach. See base.h for the license. */ + +#ifndef _MACH_C17_IPC_H +#define _MACH_C17_IPC_H 1 + +#include <mach/c17/base.h> +#include <mach/c17/atomic.h> +#include <mach/c17/vm.h> + +#define MACH_IPC_INLINE_MAX 64u +#define MACH_IPC_QUEUE_DEPTH 64u +#define MACH_IPC_PORT_SET_SIZE 32u +#define MACH_IPC_NAME_TABLE_SIZE 256u + +/* Mach message bits, preserved so that existing MIG stubs keep their + wire format. */ +#define MACH_MSGH_BITS_REMOTE_MASK 0x000000ffu +#define MACH_MSGH_BITS_LOCAL_MASK 0x0000ff00u +#define MACH_MSGH_BITS_COMPLEX 0x80000000u +#define MACH_MSGH_BITS_OOL 0x40000000u + +#define MACH_IPC_MSG_INLINE 0x0u +#define MACH_IPC_MSG_OOL 0x1u + +struct mach_msg_header +{ + mach_port_name_t msgh_remote_port; + mach_port_name_t msgh_local_port; + mach_msg_size_t msgh_size; + mach_msg_id_t msgh_id; + uint32_t msgh_bits; +}; + +/* An out-of-line region is the zero-copy payload of a large message. + The sender keeps one reference, the message queue owns one, and the + receiver obtains one when it maps the region. No byte is copied. */ +struct mach_ipc_ool +{ + mach_refcount_t refs; + mach_phys_t address; + size_t length; + uint32_t protection; + mach_port_name_t owner; +}; + +struct mach_msg +{ struct mach_msg_header header; + uint32_t kind; + union + { + struct + { + uint8_t bytes[MACH_IPC_INLINE_MAX]; + } inl; + struct + { + struct mach_ipc_ool *region; + uint64_t offset; + } ool; + } data; +}; + +/* Single-producer/single-consumer ring for message descriptors. The + head and tail live on separate cache lines; the descriptor itself is + copied, the payload never is. */ +struct mach_ipc_queue +{ + MACH_CACHELINE_ALIGN mach_atomic_u32 head; + MACH_CACHELINE_ALIGN mach_atomic_u32 tail; + MACH_CACHELINE_ALIGN mach_atomic_u32 dropped; + MACH_CACHELINE_ALIGN mach_atomic_u32 sequence[MACH_IPC_QUEUE_DEPTH]; + MACH_CACHELINE_ALIGN struct mach_msg slots[MACH_IPC_QUEUE_DEPTH]; +}; + +struct mach_ipc_port; + +struct mach_ipc_port_ops +{ + /* Map an out-of-line region into the receiver's address space. */ mach_error_t (*map_ool) (void *context, struct mach_ipc_ool *region, + mach_vaddr_t *address); + /* Called when a port dies, so that blocked senders can be told. */ + void (*port_dead) (void *context, struct mach_ipc_port *port); +}; + +struct mach_ipc_port +{ + mach_port_name_t name; + mach_refcount_t refs; + struct mach_spinlock lock; + void *context; + const struct mach_ipc_port_ops *ops; + MACH_CACHELINE_ALIGN struct mach_ipc_queue queue; + MACH_CACHELINE_ALIGN mach_atomic_u32 receivers; + mach_atomic_bool dead; + mach_port_name_t notify; + uint32_t qlimit; +} MACH_CACHELINE_ALIGN; + +struct mach_ipc_port_set +{ + struct mach_ipc_port *ports[MACH_IPC_PORT_SET_SIZE]; + uint32_t count; + uint32_t next; + struct mach_spinlock lock; +}; + +struct mach_ipc_entry +{ + mach_port_name_t name; + struct mach_ipc_port *port; + uint32_t rights; +}; + +struct mach_ipc_space +{ + struct mach_ipc_entry entries[MACH_IPC_NAME_TABLE_SIZE]; + uint32_t count; + mach_atomic_u32 next_name; + struct mach_spinlock lock; +}; + +/* Port lifecycle. Ports are caller-allocated so that the IPC path + never allocates on the send fast path. */ +mach_error_t mach_ipc_port_init (struct mach_ipc_port *port, + mach_port_name_t name, + const struct mach_ipc_port_ops *ops, + void *context); +void mach_ipc_port_deallocate (struct mach_ipc_port *port); + +/* Message sending and receiving. Sending an inline message copies at + most MACH_IPC_INLINE_MAX bytes; sending an OOL message transfers a + reference and copies only the descriptor. */ +mach_error_t mach_ipc_send (struct mach_ipc_port *port, + const struct mach_msg *message); +mach_error_t mach_ipc_receive (struct mach_ipc_port *port, + struct mach_msg *message, bool blocking); +mach_error_t mach_ipc_receive_set (struct mach_ipc_port_set *set, + struct mach_msg *message, bool blocking); + +/* Convenience constructors. */ +void mach_ipc_msg_init_inline (struct mach_msg *message, + mach_port_name_t remote, + mach_port_name_t local, mach_msg_id_t id, + const void *bytes, size_t length); +void mach_ipc_msg_init_ool (struct mach_msg *message, + mach_port_name_t remote, mach_port_name_t local, + mach_msg_id_t id, struct mach_ipc_ool *region, + uint64_t offset, size_t length); +struct mach_ipc_ool *mach_ipc_ool_create (mach_phys_t address, size_t length, + uint32_t protection); +void mach_ipc_ool_reference (struct mach_ipc_ool *region); +void mach_ipc_ool_release (struct mach_ipc_ool *region); +/* Map the region into a receiver and consume the message's reference. */ +mach_error_t mach_ipc_ool_map (struct mach_ipc_ool *region, + struct mach_vm_map *map, + mach_vaddr_t *address); + +/* Name spaces. */ +mach_error_t mach_ipc_space_init (struct mach_ipc_space *space); +mach_error_t mach_ipc_space_insert (struct mach_ipc_space *space, + struct mach_ipc_port *port, + mach_port_name_t *name); +struct mach_ipc_port *mach_ipc_space_lookup (struct mach_ipc_space *space, + mach_port_name_t name); +mach_error_t mach_ipc_space_remove (struct mach_ipc_space *space, + mach_port_name_t name); + +void mach_ipc_port_set_init (struct mach_ipc_port_set *set); +mach_error_t mach_ipc_port_set_insert (struct mach_ipc_port_set *set, + struct mach_ipc_port *port); +mach_error_t mach_ipc_port_set_remove (struct mach_ipc_port_set *set, + struct mach_ipc_port *port); + +#endif /* _MACH_C17_IPC_H */ diff --git a/modern/src/ipc/ipc.c b/modern/src/ipc/ipc.c new file mode 100644 index 00000000..478b010b --- /dev/null +++ b/modern/src/ipc/ipc.c @@ -0,0 +1,542 @@ +/* Mach IPC: cache-friendly ports and zero-copy out-of-line messages. + Copyright (C) 2026 Free Software Foundation, Inc. + + This file is part of GNU Mach. See base.h for the license. + + The port queue is a bounded Vyukov ring. Each slot carries its own + sequence number, which makes the ring safe for several concurrent + senders while keeping the receive path a single load and store. The + head and tail indices live on separate cache lines so that a sender + and a receiver never invalidate each other's line. */ + +#include <threads.h> + +#include <mach/c17/ipc.h> + +#define MACH_IPC_OOL_POOL 128u + +static struct mach_ipc_ool mach_ool_pool[MACH_IPC_OOL_POOL]; +static mach_atomic_bool mach_ool_used[MACH_IPC_OOL_POOL]; +static mach_atomic_u32 mach_ool_pool_ready; + +static void +ipc_ool_pool_init (void) +{ + uint32_t index; + + if (atomic_load_explicit (&mach_ool_pool_ready, memory_order_acquire)) + return; + + for (index = 0; index < MACH_IPC_OOL_POOL; index++) + { + atomic_init (&mach_ool_used[index], false); + mach_refcount_init (&mach_ool_pool[index].refs, 0); + } + + atomic_store_explicit (&mach_ool_pool_ready, 1u, memory_order_release); +} + +static void +ipc_queue_init (struct mach_ipc_queue *queue) +{ + uint32_t index; + + atomic_init (&queue->head, 0u); + atomic_init (&queue->tail, 0u); + atomic_init (&queue->dropped, 0u); + + for (index = 0; index < MACH_IPC_QUEUE_DEPTH; index++) + atomic_init (&queue->sequence[index], index); +} + +static bool +ipc_queue_push (struct mach_ipc_queue *queue, const struct mach_msg *message) +{ + uint32_t position = atomic_load_explicit (&queue->tail, + memory_order_relaxed); + + for (;;) + { + struct mach_msg *slot = &queue->slots[position % MACH_IPC_QUEUE_DEPTH]; + uint32_t sequence = atomic_load_explicit ( + &queue->sequence[position % MACH_IPC_QUEUE_DEPTH], + memory_order_acquire); + int32_t difference = (int32_t) (sequence - position); + + if (difference == 0) + { + if (atomic_compare_exchange_weak_explicit (&queue->tail, &position, + position + 1u, + memory_order_relaxed, + memory_order_relaxed)) + { + *slot = *message; + atomic_store_explicit ( + &queue->sequence[position % MACH_IPC_QUEUE_DEPTH], + position + 1u, memory_order_release); + return true; + } + } + else if (difference < 0) + { + atomic_fetch_add_explicit (&queue->dropped, 1u, + memory_order_relaxed); + return false; + } + else + position = atomic_load_explicit (&queue->tail, memory_order_relaxed); + } +} + +static bool +ipc_queue_pop (struct mach_ipc_queue *queue, struct mach_msg *message) +{ + uint32_t position = atomic_load_explicit (&queue->head, + memory_order_relaxed); + + for (;;) + { + uint32_t sequence = atomic_load_explicit ( + &queue->sequence[position % MACH_IPC_QUEUE_DEPTH], + memory_order_acquire); + int32_t difference = (int32_t) (sequence - (position + 1u)); + + if (difference == 0) + { + if (atomic_compare_exchange_weak_explicit (&queue->head, &position, + position + 1u, + memory_order_relaxed, + memory_order_relaxed)) + { + *message = queue->slots[position % MACH_IPC_QUEUE_DEPTH]; + atomic_store_explicit ( + &queue->sequence[position % MACH_IPC_QUEUE_DEPTH], + position + MACH_IPC_QUEUE_DEPTH, memory_order_release); + return true; + } + } + else if (difference < 0) + return false; + else + position = atomic_load_explicit (&queue->head, memory_order_relaxed); + } +} + +mach_error_t +mach_ipc_port_init (struct mach_ipc_port *port, mach_port_name_t name, + const struct mach_ipc_port_ops *ops, void *context) +{ + if (port == NULL) + return MACH_EINVAL; + + mach_memset (port, 0, sizeof (*port)); + port->name = name; + port->ops = ops; + port->context = context; + port->qlimit = MACH_IPC_QUEUE_DEPTH; + mach_refcount_init (&port->refs, 1u); + mach_spinlock_init (&port->lock); + ipc_queue_init (&port->queue); + atomic_init (&port->receivers, 0u); + atomic_init (&port->dead, false); + return MACH_OK; +} + +void +mach_ipc_port_deallocate (struct mach_ipc_port *port) +{ + struct mach_msg message; + + if (port == NULL || !mach_refcount_dec_and_test (&port->refs)) + return; + + atomic_store_explicit (&port->dead, true, memory_order_release); + + while (ipc_queue_pop (&port->queue, &message)) + if (message.kind == MACH_IPC_MSG_OOL && message.data.ool.region != NULL) + mach_ipc_ool_release (message.data.ool.region); + + if (port->ops != NULL && port->ops->port_dead != NULL) + port->ops->port_dead (port->context, port); +} + +mach_error_t +mach_ipc_send (struct mach_ipc_port *port, const struct mach_msg *message) +{ + mach_error_t error; + + if (port == NULL || message == NULL) + return MACH_EINVAL; + + if (atomic_load_explicit (&port->dead, memory_order_acquire)) + return MACH_ENODEV; + + if (message->kind == MACH_IPC_MSG_OOL) + { + if (message->data.ool.region == NULL + || message->data.ool.region->length == 0) + return MACH_EINVAL; + mach_ipc_ool_reference (message->data.ool.region); + } + + mach_spinlock_lock (&port->lock); + if (!ipc_queue_push (&port->queue, message)) + { + mach_spinlock_unlock (&port->lock); + if (message->kind == MACH_IPC_MSG_OOL) + mach_ipc_ool_release (message->data.ool.region); + return MACH_EAGAIN; + } + mach_spinlock_unlock (&port->lock); + + error = MACH_OK; + return error; +} + +mach_error_t +mach_ipc_receive (struct mach_ipc_port *port, struct mach_msg *message, + bool blocking) +{ + if (port == NULL || message == NULL) + return MACH_EINVAL; + + for (;;) + { + mach_spinlock_lock (&port->lock); + if (ipc_queue_pop (&port->queue, message)) + { + mach_spinlock_unlock (&port->lock); + return MACH_OK; + } + mach_spinlock_unlock (&port->lock); + + if (atomic_load_explicit (&port->dead, memory_order_acquire)) + return MACH_ENODEV; + + if (!blocking) + return MACH_EAGAIN; + + thrd_yield (); + } +} + +void +mach_ipc_port_set_init (struct mach_ipc_port_set *set) +{ + mach_memset (set, 0, sizeof (*set)); + mach_spinlock_init (&set->lock); +} + +mach_error_t +mach_ipc_port_set_insert (struct mach_ipc_port_set *set, + struct mach_ipc_port *port) +{ + if (set == NULL || port == NULL || set->count >= MACH_IPC_PORT_SET_SIZE) + return MACH_ENOMEM; + + mach_spinlock_lock (&set->lock); + for (uint32_t index = 0; index < set->count; index++) + if (set->ports[index] == port) + { + mach_spinlock_unlock (&set->lock); + return MACH_EBUSY; + } + + set->ports[set->count++] = port; + mach_refcount_inc (&port->refs); + mach_spinlock_unlock (&set->lock); + return MACH_OK; +} + +mach_error_t +mach_ipc_port_set_remove (struct mach_ipc_port_set *set, + struct mach_ipc_port *port) +{ + if (set == NULL || port == NULL) + return MACH_EINVAL; + + mach_spinlock_lock (&set->lock); + for (uint32_t index = 0; index < set->count; index++) + if (set->ports[index] == port) + { + for (uint32_t move = index + 1; move < set->count; move++) + set->ports[move - 1] = set->ports[move]; + set->count--; + mach_spinlock_unlock (&set->lock); + mach_ipc_port_deallocate (port); + return MACH_OK; + } + mach_spinlock_unlock (&set->lock); + return MACH_ENOENT; +} + +mach_error_t +mach_ipc_receive_set (struct mach_ipc_port_set *set, struct mach_msg *message, + bool blocking) +{ + if (set == NULL || message == NULL) + return MACH_EINVAL; + + for (;;) + { + uint32_t start; + uint32_t count; + + mach_spinlock_lock (&set->lock); + start = set->next; + count = set->count; + mach_spinlock_unlock (&set->lock); + + for (uint32_t index = 0; index < count; index++) + { + uint32_t slot = (start + index) % count; + struct mach_ipc_port *port = set->ports[slot]; + + if (mach_ipc_receive (port, message, false) == MACH_OK) + { + mach_spinlock_lock (&set->lock); + set->next = (slot + 1u) % set->count; + mach_spinlock_unlock (&set->lock); + return MACH_OK; + } + } + + if (!blocking) + return MACH_EAGAIN; + + thrd_yield (); + } +} + +void +mach_ipc_msg_init_inline (struct mach_msg *message, mach_port_name_t remote, + mach_port_name_t local, mach_msg_id_t id, + const void *bytes, size_t length) +{ + mach_memset (message, 0, sizeof (*message)); + message->header.msgh_remote_port = remote; + message->header.msgh_local_port = local; + message->header.msgh_id = id; + message->header.msgh_size = (mach_msg_size_t) length; + message->header.msgh_bits = 0; + message->kind = MACH_IPC_MSG_INLINE; + + if (length > MACH_IPC_INLINE_MAX) + length = MACH_IPC_INLINE_MAX; + + if (bytes != NULL) + mach_memcpy (message->data.inl.bytes, bytes, length); +} + +void +mach_ipc_msg_init_ool (struct mach_msg *message, mach_port_name_t remote, + mach_port_name_t local, mach_msg_id_t id, + struct mach_ipc_ool *region, uint64_t offset, + size_t length) +{ + mach_memset (message, 0, sizeof (*message)); + message->header.msgh_remote_port = remote; + message->header.msgh_local_port = local; + message->header.msgh_id = id; + message->header.msgh_size = (mach_msg_size_t) length; + message->header.msgh_bits = MACH_MSGH_BITS_COMPLEX | MACH_MSGH_BITS_OOL; + message->kind = MACH_IPC_MSG_OOL; + message->data.ool.region = region; + message->data.ool.offset = offset; +} + +struct mach_ipc_ool * +mach_ipc_ool_create (mach_phys_t address, size_t length, uint32_t protection) +{ + uint32_t index; + + ipc_ool_pool_init (); + + for (index = 0; index < MACH_IPC_OOL_POOL; index++) + if (!atomic_exchange_explicit (&mach_ool_used[index], true, + memory_order_acquire)) + { + struct mach_ipc_ool *region = &mach_ool_pool[index]; + + mach_refcount_init (®ion->refs, 1u); + region->address = address; + region->length = length; + region->protection = protection; + region->owner = MACH_PORT_NULL; + return region; + } + + return NULL; +} + +void +mach_ipc_ool_reference (struct mach_ipc_ool *region) +{ + if (region != NULL) + mach_refcount_inc (®ion->refs); +} + +void +mach_ipc_ool_release (struct mach_ipc_ool *region) +{ + size_t index; + + if (region == NULL || !mach_refcount_dec_and_test (®ion->refs)) + return; + + index = (size_t) (region - mach_ool_pool); + if (index < MACH_IPC_OOL_POOL) + atomic_store_explicit (&mach_ool_used[index], false, + memory_order_release); +} + +mach_error_t +mach_ipc_ool_map (struct mach_ipc_ool *region, struct mach_vm_map *map, + mach_vaddr_t *address) +{ + mach_error_t error; + mach_vaddr_t hint; + size_t length; + + if (region == NULL || map == NULL || address == NULL + || region->length == 0) + return MACH_EINVAL; + + hint = mach_align_up_u64 (*address != 0 ? *address : map->next_hint, + MACH_PAGE_SIZE); + length = (size_t) mach_align_up_u64 (region->length, MACH_PAGE_SIZE); + + error = mach_pmap_map (&map->pmap, hint, region->address, length, + MACH_VM_PROT_READ | MACH_VM_PROT_WRITE + | MACH_VM_PROT_USER); + if (error != MACH_OK) + return error; + + map->next_hint = hint + length; + *address = hint; + + /* Mapping consumes the reference carried by the received message. */ + mach_ipc_ool_release (region); + return MACH_OK; +} + +mach_error_t +mach_ipc_space_init (struct mach_ipc_space *space) +{ + if (space == NULL) + return MACH_EINVAL; + + mach_memset (space, 0, sizeof (*space)); + mach_spinlock_init (&space->lock); + atomic_init (&space->next_name, 1u); + return MACH_OK; +} + +mach_error_t +mach_ipc_space_insert (struct mach_ipc_space *space, struct mach_ipc_port *port, + mach_port_name_t *name) +{ + uint32_t candidate; + uint32_t start; + + if (space == NULL || port == NULL) + return MACH_EINVAL; + + candidate = atomic_fetch_add_explicit (&space->next_name, 1u, + memory_order_relaxed); + if (candidate == 0 || candidate == MACH_PORT_DEAD) + candidate = atomic_fetch_add_explicit (&space->next_name, 1u, + memory_order_relaxed); + + mach_spinlock_lock (&space->lock); + + if (space->count >= MACH_IPC_NAME_TABLE_SIZE) + { + mach_spinlock_unlock (&space->lock); + return MACH_ENOMEM; + } + + start = candidate % MACH_IPC_NAME_TABLE_SIZE; + for (uint32_t probe = 0; probe < MACH_IPC_NAME_TABLE_SIZE; probe++) + { + uint32_t slot = (start + probe) % MACH_IPC_NAME_TABLE_SIZE; + + if (space->entries[slot].port == NULL) + { + space->entries[slot].name = candidate; + space->entries[slot].port = port; + space->entries[slot].rights = 1; + space->count++; + mach_refcount_inc (&port->refs); + mach_spinlock_unlock (&space->lock); + if (name != NULL) + *name = candidate; + return MACH_OK; + } + } + + mach_spinlock_unlock (&space->lock); + return MACH_ENOMEM; +} + +struct mach_ipc_port * +mach_ipc_space_lookup (struct mach_ipc_space *space, mach_port_name_t name) +{ + uint32_t start; + struct mach_ipc_port *port = NULL; + + if (space == NULL || name == MACH_PORT_NULL) + return NULL; + + start = name % MACH_IPC_NAME_TABLE_SIZE; + mach_spinlock_lock (&space->lock); + + for (uint32_t probe = 0; probe < MACH_IPC_NAME_TABLE_SIZE; probe++) + { + struct mach_ipc_entry *entry + = &space->entries[(start + probe) % MACH_IPC_NAME_TABLE_SIZE]; + + if (entry->port != NULL && entry->name == name) + { + port = entry->port; + mach_refcount_inc (&port->refs); + break; + } + } + + mach_spinlock_unlock (&space->lock); + return port; +} + +mach_error_t +mach_ipc_space_remove (struct mach_ipc_space *space, mach_port_name_t name) +{ + uint32_t start; + + if (space == NULL || name == MACH_PORT_NULL) + return MACH_EINVAL; + + start = name % MACH_IPC_NAME_TABLE_SIZE; + mach_spinlock_lock (&space->lock); + + for (uint32_t probe = 0; probe < MACH_IPC_NAME_TABLE_SIZE; probe++) + { + struct mach_ipc_entry *entry + = &space->entries[(start + probe) % MACH_IPC_NAME_TABLE_SIZE]; + + if (entry->port != NULL && entry->name == name) + { + struct mach_ipc_port *port = entry->port; + + entry->port = NULL; + entry->name = MACH_PORT_NULL; + space->count--; + mach_spinlock_unlock (&space->lock); + mach_ipc_port_deallocate (port); + return MACH_OK; + } + } + + mach_spinlock_unlock (&space->lock); + return MACH_ENOENT; +} diff --git a/modern/tests/test_ipc.c b/modern/tests/test_ipc.c new file mode 100644 index 00000000..bb8aea59 --- /dev/null +++ b/modern/tests/test_ipc.c @@ -0,0 +1,189 @@ +/* Tests for ports, the cache-friendly ring and zero-copy messages. */ + +#include <mach/c17/ipc.h> + +#include "test.h" + +#define ARENA_SIZE (4u * 1024u * 1024u) + +static unsigned char arena[ARENA_SIZE] __attribute__ ((aligned (4096))); +static mach_phys_t table_cursor = 0x200000u; + +static void * +fake_phys_to_virt (void *context, mach_phys_t address) +{ + (void) context; + return arena + address; +} + +static mach_phys_t +fake_alloc_table (void *context) +{ + mach_phys_t result = table_cursor; + + (void) context; + if (table_cursor + MACH_PAGE_SIZE > ARENA_SIZE) + return MACH_PHYS_NULL; + table_cursor += MACH_PAGE_SIZE; + return result; +} + +static void +fake_free (void *context, mach_phys_t address) +{ + (void) context; + (void) address; +} + +static void +fake_flush (void *context, mach_vaddr_t address, size_t length) +{ + (void) context; + (void) address; + (void) length; +} + +static struct mach_pmap_ops fake_ops = +{ + .phys_to_virt = fake_phys_to_virt, + .alloc_table = fake_alloc_table, + .free_table = fake_free, + .flush_tlb = fake_flush +}; + +static struct mach_ipc_port port; + +static void +test_layout (void) +{ + CHECK_EQ (_Alignof (struct mach_ipc_port) >= MACH_CACHELINE, 1); + CHECK_EQ (offsetof (struct mach_ipc_port, queue.head) % MACH_CACHELINE, 0); + CHECK_EQ (offsetof (struct mach_ipc_port, queue.tail) % MACH_CACHELINE, 0); + CHECK_EQ (offsetof (struct mach_ipc_port, receivers) % MACH_CACHELINE, 0); +} + +static void +test_inline (void) +{ + struct mach_msg message; + struct mach_msg received; + uint32_t payload = 0xdeadbeefu; + + CHECK_EQ (mach_ipc_port_init (&port, 1, NULL, NULL), MACH_OK); + CHECK_EQ (mach_ipc_receive (&port, &received, false), MACH_EAGAIN); + + mach_ipc_msg_init_inline (&message, 1, 2, 7, &payload, sizeof (payload)); + CHECK_EQ (mach_ipc_send (&port, &message), MACH_OK); + CHECK_EQ (mach_ipc_receive (&port, &received, false), MACH_OK); + CHECK_EQ (received.header.msgh_id, 7); + CHECK_EQ (received.header.msgh_size, sizeof (payload)); + CHECK_EQ (*(uint32_t *) received.data.inl.bytes, payload); + + for (uint32_t index = 0; index < MACH_IPC_QUEUE_DEPTH; index++) + { + message.header.msgh_id = (mach_msg_id_t) index; + CHECK_EQ (mach_ipc_send (&port, &message), MACH_OK); + } + CHECK_EQ (mach_ipc_send (&port, &message), MACH_EAGAIN); + CHECK_EQ (atomic_load_explicit (&port.queue.dropped, memory_order_relaxed), + 1); + + for (uint32_t index = 0; index < MACH_IPC_QUEUE_DEPTH; index++) + { + CHECK_EQ (mach_ipc_receive (&port, &received, false), MACH_OK); + CHECK_EQ (received.header.msgh_id, (mach_msg_id_t) index); + } +} + +static void +test_zero_copy (void) +{ + struct mach_vm_slot slots[1]; + struct mach_vm_resident residents[1]; + struct mach_vm_object object; + struct mach_vm_map map; + struct mach_pmap_format format; + struct mach_ipc_ool *region; + struct mach_msg message; + struct mach_msg received; + mach_vaddr_t address = 0; + mach_phys_t translated; + + memset (arena + 0x100000, 0x5a, 0x1000); + + mach_pmap_format_init (&format, MACH_PMAP_X86_64, 4); + CHECK_EQ (mach_vm_object_init (&object, slots, 1, residents, 1, NULL, 4096), + MACH_OK); + CHECK_EQ (mach_vm_map_init (&map, &object, NULL, &format, &fake_ops, NULL, + MACH_VM_PROT_READ | MACH_VM_PROT_WRITE), + MACH_OK); + + region = mach_ipc_ool_create (0x100000u, 4096, MACH_VM_PROT_READ); + CHECK (region != NULL); + CHECK_EQ (mach_refcount_load (®ion->refs), 1); + + CHECK_EQ (mach_ipc_port_init (&port, 2, NULL, NULL), MACH_OK); + mach_ipc_msg_init_ool (&message, 2, 3, 9, region, 0, 4096); + CHECK_EQ (mach_ipc_send (&port, &message), MACH_OK); + CHECK_EQ (mach_refcount_load (®ion->refs), 2); + mach_ipc_ool_release (region); + CHECK_EQ (mach_refcount_load (®ion->refs), 1); + + CHECK_EQ (mach_ipc_receive (&port, &received, false), MACH_OK); + CHECK_EQ (received.kind, MACH_IPC_MSG_OOL); + CHECK_EQ (received.data.ool.region, region); + CHECK_EQ ((received.header.msgh_bits & MACH_MSGH_BITS_OOL) != 0, 1); + + address = 0x40000000u; + CHECK_EQ (mach_ipc_ool_map (received.data.ool.region, &map, &address), + MACH_OK); + CHECK_EQ (address, 0x40000000u); + CHECK_EQ (mach_pmap_translate (&map.pmap, address, &translated, NULL), + MACH_OK); + CHECK_EQ (translated, 0x100000u); + CHECK_EQ (arena[0x100000], 0x5a); +} + +static void +test_port_set_and_space (void) +{ + struct mach_ipc_port ports[2]; + struct mach_ipc_port_set set; + struct mach_ipc_space space; + struct mach_msg message; + struct mach_msg received; + mach_port_name_t name; + + CHECK_EQ (mach_ipc_port_init (&ports[0], 10, NULL, NULL), MACH_OK); + CHECK_EQ (mach_ipc_port_init (&ports[1], 11, NULL, NULL), MACH_OK); + + mach_ipc_port_set_init (&set); + CHECK_EQ (mach_ipc_port_set_insert (&set, &ports[0]), MACH_OK); + CHECK_EQ (mach_ipc_port_set_insert (&set, &ports[1]), MACH_OK); + CHECK_EQ (mach_ipc_port_set_insert (&set, &ports[1]), MACH_EBUSY); + + /* Messages alternate across the set in round-robin order. */ + mach_ipc_msg_init_inline (&message, 10, 20, 1, NULL, 0); + CHECK_EQ (mach_ipc_send (&ports[1], &message), MACH_OK); + CHECK_EQ (mach_ipc_receive_set (&set, &received, false), MACH_OK); + CHECK_EQ (received.header.msgh_id, 1); + + CHECK_EQ (mach_ipc_space_init (&space), MACH_OK); + CHECK_EQ (mach_ipc_space_insert (&space, &ports[0], &name), MACH_OK); + CHECK (name != MACH_PORT_NULL); + CHECK (mach_ipc_space_lookup (&space, name) == &ports[0]); + CHECK (mach_ipc_space_lookup (&space, name + 1000u) == NULL); + CHECK_EQ (mach_ipc_space_remove (&space, name), MACH_OK); + CHECK (mach_ipc_space_lookup (&space, name) == NULL); +} + +static void +run_tests (void) +{ + test_layout (); + test_inline (); + test_zero_copy (); + test_port_set_and_space (); +} + +TEST_MAIN ("ipc") -- 2.56.0
