The branch main has been updated by adrian: URL: https://cgit.FreeBSD.org/src/commit/?id=9f90536c74b8172fc67cd977e5451f37a12462d5
commit 9f90536c74b8172fc67cd977e5451f37a12462d5 Author: Abdelkader Boudih <[email protected]> AuthorDate: 2026-06-14 20:54:28 +0000 Commit: Adrian Chadd <[email protected]> CommitDate: 2026-06-14 20:56:53 +0000 apple_bce/vhci: add T2 virtual USB host controller Implements a VHCI driver on top of the BCE transport: - Virtual USB bus registration via usb_controller - Port discovery and device enumeration - Control, interrupt, and bulk endpoint support - Firmware event handling with taskqueue - Suspend/resume via BCE mailbox Provides keyboard, trackpad, and Touch Bar access on T2 Macs. Tested-on: MacBookPro16,2 (A2251), Mac mini 8,1 (A1993) Reviewed by: adrian Differential Revision: https://reviews.freebsd.org/D57089 --- sys/conf/files.amd64 | 1 + sys/dev/apple_bce/apple_bce.c | 87 + sys/dev/apple_bce/apple_bce.h | 3 +- sys/dev/apple_bce/apple_bce_mailbox.c | 5 + sys/dev/apple_bce/apple_bce_vhci.c | 4821 +++++++++++++++++++++++++++++++ sys/dev/apple_bce/apple_bce_vhci.h | 251 ++ sys/dev/usb/controller/usb_controller.c | 1 + sys/modules/apple_bce/Makefile | 2 + 8 files changed, 5170 insertions(+), 1 deletion(-) diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index 4d37266d1973..0b4a22eebc78 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -115,6 +115,7 @@ dev/amdgpio/amdgpio.c optional amdgpio dev/apple_bce/apple_bce.c optional apple_bce pci dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci dev/apple_bce/apple_bce_queue.c optional apple_bce pci +dev/apple_bce/apple_bce_vhci.c optional apple_bce pci dev/asmc/asmc.c optional asmc isa dev/asmc/asmcmmio.c optional asmc isa dev/axgbe/if_axgbe_pci.c optional axp diff --git a/sys/dev/apple_bce/apple_bce.c b/sys/dev/apple_bce/apple_bce.c index 8700eddbc0fe..2b0e6928ffcd 100644 --- a/sys/dev/apple_bce/apple_bce.c +++ b/sys/dev/apple_bce/apple_bce.c @@ -32,10 +32,13 @@ #include "apple_bce.h" #include "apple_bce_mailbox.h" #include "apple_bce_queue.h" +#include "apple_bce_vhci.h" static int apple_bce_probe(device_t dev); static int apple_bce_attach(device_t dev); static int apple_bce_detach(device_t dev); +static int apple_bce_suspend(device_t dev); +static int apple_bce_resume(device_t dev); static void apple_bce_timestamp_cb(void *arg); static void apple_bce_timestamp_init(struct apple_bce_softc *sc); static void apple_bce_timestamp_start(struct apple_bce_softc *sc, @@ -516,6 +519,12 @@ apple_bce_attach(device_t dev) goto fail; device_printf(dev, "Apple T2 BCE initialized\n"); + + /* Create VHCI child for virtual USB */ + error = bce_vhci_attach(sc); + if (error != 0) + goto fail; + return (0); fail: @@ -531,6 +540,9 @@ apple_bce_detach(device_t dev) { struct apple_bce_softc *sc = device_get_softc(dev); + /* 0. Detach VHCI child first (before destroying parent resources) */ + bce_vhci_detach(sc); + /* 1. Stop timestamp */ if (sc->sc_bar4 != NULL && mtx_initialized(&sc->sc_timestamp_lock)) apple_bce_timestamp_stop(sc); @@ -616,10 +628,85 @@ apple_bce_detach(device_t dev) return (0); } +static int +apple_bce_suspend(device_t dev) +{ + struct apple_bce_softc *sc = device_get_softc(dev); + int error, restore_error; + + apple_bce_timestamp_stop(sc); + + error = bce_vhci_detach(sc); + if (error != 0) { + device_printf(dev, "failed to detach VHCI for suspend: %d\n", + error); + apple_bce_timestamp_start(sc, 0); + return (error); + } + + error = bce_mailbox_send(&sc->sc_mbox, + BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), NULL, + BCE_MBOX_TIMEOUT_MS); + if (error != 0) { + device_printf(dev, + "failed to send SLEEP_NO_STATE mailbox command: %d\n", + error); + restore_error = bce_vhci_attach(sc); + if (restore_error != 0) { + device_printf(dev, + "failed to reattach VHCI after suspend error: %d\n", + restore_error); + } + apple_bce_timestamp_start(sc, 0); + return (error); + } + + return (0); +} + +static int +apple_bce_resume(device_t dev) +{ + struct apple_bce_softc *sc = device_get_softc(dev); + uint64_t reply; + int error; + + error = bce_mailbox_send(&sc->sc_mbox, + BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &reply, + BCE_MBOX_TIMEOUT_MS); + if (error != 0) { + device_printf(dev, + "failed to send RESTORE_NO_STATE mailbox command: %d\n", + error); + return (error); + } + + if (BCE_MB_TYPE(reply) != BCE_MB_RESTORE_NO_STATE) { + device_printf(dev, + "unexpected RESTORE_NO_STATE reply: type=%u val=0x%llx\n", + BCE_MB_TYPE(reply), + (unsigned long long)BCE_MB_VALUE(reply)); + return (EINVAL); + } + + error = bce_vhci_attach(sc); + if (error != 0) { + device_printf(dev, "failed to reattach VHCI after resume: %d\n", + error); + apple_bce_timestamp_start(sc, 0); + return (error); + } + + apple_bce_timestamp_start(sc, 0); + return (0); +} + static device_method_t apple_bce_methods[] = { DEVMETHOD(device_probe, apple_bce_probe), DEVMETHOD(device_attach, apple_bce_attach), DEVMETHOD(device_detach, apple_bce_detach), + DEVMETHOD(device_suspend, apple_bce_suspend), + DEVMETHOD(device_resume, apple_bce_resume), DEVMETHOD_END }; diff --git a/sys/dev/apple_bce/apple_bce.h b/sys/dev/apple_bce/apple_bce.h index 7c2198c7232b..0e1999e48d5b 100644 --- a/sys/dev/apple_bce/apple_bce.h +++ b/sys/dev/apple_bce/apple_bce.h @@ -27,7 +27,7 @@ #define BCE_PCI_DEVICE_T2 0x1801 #define BCE_MAX_QUEUE_COUNT 0x100 -#define BCE_MAX_CQ_COUNT 16 /* Max completion queues tracked */ +#define BCE_MAX_CQ_COUNT 64 /* Max completion queues tracked */ #define BCE_QUEUE_USER_MIN 2 #define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) #define BCE_CMD_SIZE 0x40 @@ -286,6 +286,7 @@ struct apple_bce_softc { struct mtx sc_queues_lock; struct bce_queue_cq *sc_cq_list[BCE_MAX_CQ_COUNT]; struct bce_queue_sq *sc_int_sq_list[BCE_MAX_QUEUE_COUNT]; + device_t sc_vhci_dev; }; /* Inline helpers */ diff --git a/sys/dev/apple_bce/apple_bce_mailbox.c b/sys/dev/apple_bce/apple_bce_mailbox.c index abdb809b5c54..c19a01b7269c 100644 --- a/sys/dev/apple_bce/apple_bce_mailbox.c +++ b/sys/dev/apple_bce/apple_bce_mailbox.c @@ -57,6 +57,11 @@ bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv, bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 8, 0); bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 12, 0); + if (recv == NULL) { + atomic_store_int(&mb->status, 0); + return (0); + } + /* Wait for interrupt-driven reply */ if (sema_timedwait(&mb->mb_cmpl, hz * timeout_ms / 1000) != 0) { /* Timeout -- reset to idle */ diff --git a/sys/dev/apple_bce/apple_bce_vhci.c b/sys/dev/apple_bce/apple_bce_vhci.c new file mode 100644 index 000000000000..b8dbc9638b36 --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_vhci.c @@ -0,0 +1,4821 @@ +/*- + * Copyright (c) 2026 Abdelkader Boudih <[email protected]> + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple T2 BCE Virtual USB Host Controller Interface (VHCI). + */ + +#ifdef USB_GLOBAL_INCLUDE_FILE +#include USB_GLOBAL_INCLUDE_FILE +#else +#include <sys/stdint.h> +#include <sys/stddef.h> +#include <sys/param.h> +#include <sys/queue.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/module.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/condvar.h> +#include <sys/sysctl.h> +#include <sys/sx.h> +#include <sys/unistd.h> +#include <sys/callout.h> +#include <sys/malloc.h> +#include <sys/priv.h> +#endif + +#include <sys/sema.h> +#include <sys/taskqueue.h> +#include <sys/endian.h> +#include <machine/bus.h> +#include <machine/atomic.h> + +#include <dev/usb/usb.h> +#include <dev/usb/usbdi.h> +#include <dev/usb/usb_core.h> +#include <dev/usb/usb_busdma.h> +#include <dev/usb/usb_process.h> +#include <dev/usb/usb_transfer.h> +#include <dev/usb/usb_device.h> +#include <dev/usb/usb_hub.h> +#include <dev/usb/usb_util.h> +#include <dev/usb/usb_controller.h> +#include <dev/usb/usb_bus.h> + +#include "apple_bce.h" +#include "apple_bce_queue.h" +#include "apple_bce_vhci.h" + +/* + * VHCI softc, defined here because it depends on USB headers. + */ +struct bce_vhci_softc { + struct usb_bus sc_bus; /* Must be first */ + struct usb_device *sc_devices[BCE_VHCI_MAX_DEVICES]; + struct apple_bce_softc *sc_bce; + device_t sc_dev; + + /* Controller state */ + uint32_t sc_port_mask; + uint8_t sc_port_count; + int sc_started; + + /* Port state */ + uint32_t sc_port_status[BCE_VHCI_MAX_PORTS]; + uint32_t sc_port_change[BCE_VHCI_MAX_PORTS]; + uint8_t sc_port_power[BCE_VHCI_MAX_PORTS]; + + /* Hub scratch buffer (for descriptor/status responses) */ + uint8_t sc_hub_idata[32]; + + /* Message queues (host -> device) */ + struct bce_vhci_msg_queue msg_commands; + struct bce_vhci_msg_queue msg_system; + struct bce_vhci_msg_queue msg_isochronous; + struct bce_vhci_msg_queue msg_interrupt; + struct bce_vhci_msg_queue msg_asynchronous; + + /* Event queues (device -> host), share a single CQ */ + struct bce_queue_cq *ev_cq; + struct bce_vhci_evt_queue ev_commands; + struct bce_vhci_evt_queue ev_system; + struct bce_vhci_evt_queue ev_isochronous; + struct bce_vhci_evt_queue ev_interrupt; + struct bce_vhci_evt_queue ev_asynchronous; + + /* Command execution (synchronous, wraps msg_commands) */ + struct bce_vhci_cmd_queue cmd; + + /* Queue ID bitmap (256 bits = BCE_MAX_QUEUE_COUNT) */ + uint32_t sc_qid_bitmap[8]; + + /* Per-device state (indexed by firmware device ID) */ + struct bce_vhci_device sc_devs[BCE_VHCI_MAX_DEVICES]; + uint8_t sc_port_to_dev[BCE_VHCI_MAX_PORTS]; + + /* Deferred firmware event processing (from ev_commands) */ + struct task sc_fwevt_task; + volatile int sc_detaching; /* Teardown guard */ + + /* + * Firmware event mailbox: ISR copies events here, task processes. + * Protected by sc_fwevt_lock. Ring of BCE_VHCI_EVT_PENDING entries. + */ + struct mtx sc_fwevt_lock; +#define BCE_VHCI_FWEVT_RING (BCE_VHCI_EVT_PENDING + 1) + struct { + struct bce_vhci_message msg; + int needs_reply; + } sc_fwevt_ring[BCE_VHCI_FWEVT_RING]; + uint32_t sc_fwevt_prod; + uint32_t sc_fwevt_cons; + + /* Spinlock for msg_asynchronous writes (ISR + taskqueue context) */ + struct mtx sc_async_lock; + + /* Deferred endpoint reset (cannot sleep in pipe_start) */ + struct task sc_reset_task; + + /* Deferred endpoint create (cannot sleep in pipe_start) */ + struct task sc_create_task; + + /* Deferred port status change (ISR cannot call cmd_execute) */ + struct task sc_port_chg_task; + volatile uint32_t sc_port_chg_mask; +}; + +/* Command timeout (ticks) */ +#define BCE_VHCI_CMD_TIMEOUT_SHORT (hz * 2) +#define BCE_VHCI_CMD_TIMEOUT_LONG (hz * 30) + +static usb_handle_req_t bce_vhci_roothub_exec; +static void bce_vhci_endpoint_init(struct usb_device *udev, + struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep); +static void bce_vhci_xfer_setup(struct usb_setup_params *parm); +static void bce_vhci_xfer_unsetup(struct usb_xfer *xfer); +static void bce_vhci_get_dma_delay(struct usb_device *udev, uint32_t *pus); + +static void bce_vhci_pipe_open(struct usb_xfer *xfer); +static void bce_vhci_pipe_close(struct usb_xfer *xfer); +static void bce_vhci_pipe_enter(struct usb_xfer *xfer); +static void bce_vhci_pipe_start(struct usb_xfer *xfer); + +static int bce_vhci_probe(device_t dev); +static int bce_vhci_attach_dev(device_t dev); +static int bce_vhci_detach_dev(device_t dev); + +static int bce_vhci_alloc_qid(struct bce_vhci_softc *vhci); +static void bce_vhci_free_qid(struct bce_vhci_softc *vhci, int qid); +static int bce_vhci_create_queues(struct bce_vhci_softc *vhci); +static void bce_vhci_destroy_queues(struct bce_vhci_softc *vhci); +static int bce_vhci_start_controller(struct bce_vhci_softc *vhci); +static void bce_vhci_msg_queue_completion(struct bce_queue_sq *sq); +static void bce_vhci_ev_cmd_completion(struct bce_queue_sq *sq); +static void bce_vhci_ev_system_completion(struct bce_queue_sq *sq); +static void bce_vhci_ev_generic_completion(struct bce_queue_sq *sq); +static void bce_vhci_cmd_deliver_completion(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_handle_port_status_change(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq, uint32_t count); + +static int bce_vhci_device_create(struct bce_vhci_softc *vhci, uint8_t port); +static void bce_vhci_device_destroy(struct bce_vhci_softc *vhci, uint8_t port); +static int bce_vhci_endpoint_create(struct bce_vhci_softc *vhci, + struct bce_vhci_device *dev, uint8_t ep_addr, + struct usb_endpoint_descriptor *edesc); +static void bce_vhci_endpoint_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_device *dev, uint8_t ep_addr); +static void bce_vhci_handle_transfer_request(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_complete_ctrl_locked(struct bce_vhci_softc *vhci, + struct bce_vhci_transfer_queue *tq, struct bce_vhci_message *msg); +static void bce_vhci_handle_ctrl_status(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static uint16_t bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static uint16_t bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_fwevt_task(void *arg, int pending); +static void bce_vhci_send_fw_event_reply(struct bce_vhci_softc *vhci, + struct bce_vhci_message *req, uint16_t status); +static void bce_vhci_tq_completion(struct bce_queue_sq *sq); +static void bce_vhci_reset_task(void *arg, int pending); +static void bce_vhci_create_task(void *arg, int pending); +static void bce_vhci_port_chg_task(void *arg, int pending); +static int bce_vhci_cmd_execute(struct bce_vhci_softc *vhci, + struct bce_vhci_message *req, struct bce_vhci_message *reply, + int timeout_ticks); + +/* + * Convert USB endpoint address to tq[] index. + * ep0 (0x00) maps to index 0. For other endpoints, IN and OUT get + * separate slots: OUT 0x01 -> 1, IN 0x81 -> 2, OUT 0x02 -> 3, etc. + * Maximum index is 30 (ep 0x8F), fits in BCE_VHCI_MAX_ENDPOINTS (32). + */ +static inline uint8_t +bce_vhci_ep_index(uint8_t ep_addr) +{ + uint8_t num; + + num = ep_addr & 0x0F; + if (num == 0) + return (0); + return (num * 2 - ((ep_addr & 0x80) ? 0 : 1)); +} + +static const struct usb_bus_methods bce_vhci_bus_methods = { + .roothub_exec = bce_vhci_roothub_exec, + .endpoint_init = bce_vhci_endpoint_init, + .xfer_setup = bce_vhci_xfer_setup, + .xfer_unsetup = bce_vhci_xfer_unsetup, + .get_dma_delay = bce_vhci_get_dma_delay, +}; + +/* + * Generic pipe methods (all transfer types for now). + */ +static const struct usb_pipe_methods bce_vhci_pipe_methods = { + .open = bce_vhci_pipe_open, + .close = bce_vhci_pipe_close, + .enter = bce_vhci_pipe_enter, + .start = bce_vhci_pipe_start, +}; + +/* + * Device methods. + */ +static device_method_t bce_vhci_methods[] = { + DEVMETHOD(device_probe, bce_vhci_probe), + DEVMETHOD(device_attach, bce_vhci_attach_dev), + DEVMETHOD(device_detach, bce_vhci_detach_dev), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + + /* Bus interface for usbus child */ + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD_END +}; + +static driver_t bce_vhci_driver = { + .name = "bce_vhci", + .methods = bce_vhci_methods, + .size = sizeof(struct bce_vhci_softc), +}; + +DRIVER_MODULE(bce_vhci, apple_bce, bce_vhci_driver, 0, 0); +MODULE_DEPEND(bce_vhci, usb, 1, 1, 1); + +/* + * Hub descriptor (USB 2.0 hub with per-port power switching) + */ + +/* Hub descriptor built dynamically in roothub_exec (port count varies) */ + +/* Device descriptor for the root hub */ +static const struct usb_device_descriptor bce_vhci_devd = { + .bLength = sizeof(struct usb_device_descriptor), + .bDescriptorType = UDESC_DEVICE, + .bcdUSB = { 0x00, 0x02 }, /* USB 2.0 */ + .bDeviceClass = UDCLASS_HUB, + .bDeviceSubClass = UDSUBCLASS_HUB, + .bDeviceProtocol = UDPROTO_HSHUBSTT, + .bMaxPacketSize = 64, + .idVendor = { 0x6b, 0x10 }, /* Apple 0x106b */ + .idProduct = { 0x01, 0x18 }, /* T2 BCE 0x1801 */ + .bcdDevice = { 0x00, 0x01 }, /* 1.00 */ + .iManufacturer = 1, + .iProduct = 2, + .bNumConfigurations = 1, +}; + +static const struct usb_device_qualifier bce_vhci_odevd = { + .bLength = sizeof(struct usb_device_qualifier), + .bDescriptorType = UDESC_DEVICE_QUALIFIER, + .bcdUSB = { 0x00, 0x02 }, + .bDeviceClass = UDCLASS_HUB, + .bDeviceSubClass = UDSUBCLASS_HUB, + .bDeviceProtocol = UDPROTO_HSHUBSTT, + .bMaxPacketSize0 = 0, + .bNumConfigurations = 0, +}; + +/* Configuration descriptor + interface + endpoint */ +/* 9 + 9 + 7 = 25 bytes */ +static const uint8_t bce_vhci_confd[] = { + /* Configuration descriptor */ + 0x09, 0x02, /* bLength, bDescriptorType */ + 0x19, 0x00, /* wTotalLength = 25 */ + 0x01, 0x01, 0x00, 0xC0, 0x00, /* nIntf, cfgVal, iCfg */ + /* Interface descriptor */ + 0x09, 0x04, /* bLength, bDescriptorType */ + 0x00, 0x00, 0x01, 0x09, 0x00, 0x01, 0x00, + /* Endpoint descriptor (interrupt IN ep1) */ + 0x07, 0x05, /* bLength, bDescriptorType */ + 0x81, 0x03, 0x08, 0x00, 0xFF, /* addr, attr, maxPkt, interval */ +}; + +struct bce_vhci_dma_cb_arg { + bus_addr_t addr; + int error; +}; + +static void +bce_vhci_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct bce_vhci_dma_cb_arg *cb = arg; + + cb->error = error; + if (error == 0) + cb->addr = segs[0].ds_addr; +} + +/* + * Allocate a CQ + SQ pair and DMA message buffer for a host->device queue. + * Register it with firmware under the given name. + */ +static int +bce_vhci_msg_queue_create(struct bce_vhci_softc *vhci, + struct bce_vhci_msg_queue *mq, const char *name, int cq_qid, int sq_qid, + bce_sq_completion_fn compl_fn, void *compl_arg) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_dma_cb_arg cb; + struct bce_queue_memcfg cfg; + uint32_t el_count = BCE_VHCI_MSG_QUEUE_EL; + uint32_t status; + int error, i; + + memset(mq, 0, sizeof(*mq)); + mq->el_count = el_count; + + /* Allocate CQ */ + mq->cq = bce_alloc_cq(sc, cq_qid, el_count); + if (mq->cq == NULL) + return (ENOMEM); + + /* Register CQ with firmware via command path */ + bce_get_cq_memcfg(mq->cq, &cfg); + /* CQ interrupt vector = 4 (DMA MSI) */ + cfg.vector_or_cq = 4; + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, NULL, 0); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register CQ %d for %s: %u\n", + cq_qid, name, status); + error = EIO; + goto fail_cq; + } + + /* Register CQ in parent's dispatch tables */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = mq->cq; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == NULL) { + sc->sc_cq_list[i] = mq->cq; + break; + } + } + if (i == BCE_MAX_CQ_COUNT) { + sc->sc_queues[cq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + device_printf(vhci->sc_dev, + "CQ list full for %s\n", name); + error = ENOSPC; + goto fail_cq_reg; + } + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate SQ (element size = bce_qe_submission = 32 bytes) */ + mq->sq = bce_alloc_sq(sc, sq_qid, + sizeof(struct bce_qe_submission), el_count, + compl_fn, compl_arg); + if (mq->sq == NULL) { + error = ENOMEM; + goto fail_cq_reg; + } + + /* Register SQ with firmware under the given name */ + bce_get_sq_memcfg(mq->sq, mq->cq, &cfg); + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 1); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register SQ %d (%s): %u\n", + sq_qid, name, status); + error = EIO; + goto fail_sq; + } + + /* Register SQ in parent's dispatch tables */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = mq->sq; + sc->sc_int_sq_list[sq_qid] = mq->sq; + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate DMA-coherent message buffer */ + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, + el_count * sizeof(struct bce_vhci_message), 1, + el_count * sizeof(struct bce_vhci_message), + BUS_DMA_WAITOK, + NULL, NULL, + &mq->dma_tag); + if (error != 0) + goto fail_sq_reg; + + error = bus_dmamem_alloc(mq->dma_tag, (void **)&mq->data, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &mq->dma_map); + if (error != 0) + goto fail_dma_tag; + + error = bus_dmamap_load(mq->dma_tag, mq->dma_map, mq->data, + el_count * sizeof(struct bce_vhci_message), + bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK); + if (error != 0 || cb.error != 0) { + error = error != 0 ? error : cb.error; + goto fail_dma_mem; + } + mq->dma_addr = cb.addr; + + return (0); + +fail_dma_mem: + bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map); +fail_dma_tag: + bus_dma_tag_destroy(mq->dma_tag); +fail_sq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = NULL; + sc->sc_int_sq_list[sq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); +fail_sq: + bce_free_sq(sc, mq->sq); + mq->sq = NULL; +fail_cq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == mq->cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); +fail_cq: + bce_free_cq(sc, mq->cq); + mq->cq = NULL; + return (error); +} + +static void +bce_vhci_msg_queue_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_msg_queue *mq) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + int i; + + if (mq->cq == NULL) + return; + + /* + * Unregister and free SQ before releasing the DMA buffer it + * references + */ + if (mq->sq != NULL) { + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->sq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[mq->sq->qid] = NULL; + sc->sc_int_sq_list[mq->sq->qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + bce_free_sq(sc, mq->sq); + mq->sq = NULL; + } + + /* Free DMA message buffer */ + if (mq->data != NULL) { + bus_dmamap_unload(mq->dma_tag, mq->dma_map); + bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map); + bus_dma_tag_destroy(mq->dma_tag); + mq->data = NULL; + } + + /* Unregister and free CQ */ + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->cq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[mq->cq->qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == mq->cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); + bce_free_cq(sc, mq->cq); + mq->cq = NULL; +} + +/* + * Write a message to a host->device queue. + * Caller must have reserved a submission slot. + */ +static void +bce_vhci_msg_queue_write(struct bce_vhci_softc *vhci, + struct bce_vhci_msg_queue *mq, struct bce_vhci_message *msg) +{ + struct bce_qe_submission *s; + uint32_t sidx; + + sidx = mq->sq->tail; + s = bce_next_submission(mq->sq); + + /* Copy message into DMA buffer slot and sync for device access */ + mq->data[sidx] = *msg; + bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_PREWRITE); + + /* Fill SQ entry pointing to the DMA buffer slot */ + s->length = sizeof(struct bce_vhci_message); + s->addr = mq->dma_addr + + sidx * sizeof(struct bce_vhci_message); + s->segl_addr = 0; + s->segl_length = 0; + + bce_submit_to_device(vhci->sc_bce, mq->sq); +} + +/* + * Message queue completion: consume completions and free slots. + */ +static void +bce_vhci_msg_queue_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_msg_queue *mq = sq->userdata; + + while (sq->completion_cidx != sq->completion_tail) { + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + } + bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_POSTWRITE); +} + +/* + * Allocate an SQ (paired with the shared ev_cq) and DMA buffer for a + * device->host event queue. Register with firmware and pre-submit + * receive buffers. + */ +static int +bce_vhci_evt_queue_create(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq, const char *name, int sq_qid, + bce_sq_completion_fn compl_fn) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_dma_cb_arg cb; + struct bce_queue_memcfg cfg; + uint32_t el_count = BCE_VHCI_EVT_QUEUE_EL; + uint32_t status; + int error; + + memset(eq, 0, sizeof(*eq)); + eq->el_count = el_count; + eq->userdata = vhci; + + /* Allocate SQ (shared CQ = vhci->ev_cq) */ + eq->sq = bce_alloc_sq(sc, sq_qid, + sizeof(struct bce_qe_submission), el_count, + compl_fn, eq); + if (eq->sq == NULL) + return (ENOMEM); + + /* Register SQ with firmware (direction = from device = 0) */ + bce_get_sq_memcfg(eq->sq, vhci->ev_cq, &cfg); + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 0); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register event SQ %d (%s): %u\n", + sq_qid, name, status); + error = EIO; + goto fail_sq; + } + + /* Register SQ in dispatch tables */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = eq->sq; + sc->sc_int_sq_list[sq_qid] = eq->sq; + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate DMA-coherent receive buffer */ + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, + el_count * sizeof(struct bce_vhci_message), 1, + el_count * sizeof(struct bce_vhci_message), + BUS_DMA_WAITOK, + NULL, NULL, + &eq->dma_tag); + if (error != 0) + goto fail_sq_reg; + + error = bus_dmamem_alloc(eq->dma_tag, (void **)&eq->data, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &eq->dma_map); + if (error != 0) + goto fail_dma_tag; + + error = bus_dmamap_load(eq->dma_tag, eq->dma_map, eq->data, + el_count * sizeof(struct bce_vhci_message), + bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK); + if (error != 0 || cb.error != 0) { + error = error != 0 ? error : cb.error; + goto fail_dma_mem; + } + eq->dma_addr = cb.addr; + + /* Pre-submit receive buffers */ + bce_vhci_evt_queue_submit_pending(vhci, eq, BCE_VHCI_EVT_PENDING); + + return (0); + +fail_dma_mem: + bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map); +fail_dma_tag: + bus_dma_tag_destroy(eq->dma_tag); +fail_sq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = NULL; + sc->sc_int_sq_list[sq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); +fail_sq: + bce_free_sq(sc, eq->sq); + eq->sq = NULL; + return (error); +} + +static void +bce_vhci_evt_queue_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + + if (eq->sq == NULL) + return; + + /* Unregister SQ from dispatch tables FIRST to stop IRQ callbacks */ + bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid); + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[eq->sq->qid] = NULL; + sc->sc_int_sq_list[eq->sq->qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + + /* Now safe to free DMA buffer; no IRQ can reference it */ + if (eq->data != NULL) { + bus_dmamap_unload(eq->dma_tag, eq->dma_map); + bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map); + bus_dma_tag_destroy(eq->dma_tag); + eq->data = NULL; + } + bce_free_sq(sc, eq->sq); + eq->sq = NULL; +} + +/* + * Submit empty receive buffers to an event queue so firmware can + * write messages into them. + */ +static void +bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq, uint32_t count) +{ + struct bce_qe_submission *s; + uint32_t idx; + + bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_PREREAD); + + while (count-- > 0) { + if (bce_reserve_submission(eq->sq) != 0) { + device_printf(vhci->sc_dev, + "cannot reserve event submission\n"); + break; + } + idx = eq->sq->tail; + s = bce_next_submission(eq->sq); + s->length = sizeof(struct bce_vhci_message); + s->addr = eq->dma_addr + + idx * sizeof(struct bce_vhci_message); + s->segl_addr = 0; + s->segl_length = 0; + } + bce_submit_to_device(vhci->sc_bce, eq->sq); +} + +/* + * Enqueue a firmware event into sc_fwevt_ring for deferred processing. + * Called from ISR context; returns 0 on success, -1 if ring is full. + */ +static int +bce_vhci_fwevt_enqueue(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg, int needs_reply) +{ + uint32_t next_prod; + + mtx_lock_spin(&vhci->sc_fwevt_lock); + next_prod = (vhci->sc_fwevt_prod + 1) % BCE_VHCI_FWEVT_RING; + if (next_prod == vhci->sc_fwevt_cons) { + mtx_unlock_spin(&vhci->sc_fwevt_lock); + device_printf(vhci->sc_dev, + "fwevt ring full, dropping 0x%04x\n", msg->cmd); + return (-1); + } + vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].msg = *msg; + vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].needs_reply = needs_reply; + vhci->sc_fwevt_prod = next_prod; + mtx_unlock_spin(&vhci->sc_fwevt_lock); + + if (vhci->sc_detaching == 0) + taskqueue_enqueue(taskqueue_thread, &vhci->sc_fwevt_task); + return (0); +} + +/* + * Generic event queue completion: read messages and resubmit buffers. + * Used for system, isochronous, interrupt, and asynchronous event queues. + */ +static void +bce_vhci_ev_generic_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_evt_queue *eq = sq->userdata; + struct bce_vhci_softc *vhci = eq->userdata; + struct bce_vhci_message *msg; + uint32_t cnt = 0; + + bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_POSTREAD); + + while (sq->completion_cidx != sq->completion_tail) { + struct bce_sq_completion_data *cd; + + cd = &sq->completion_data[sq->completion_cidx]; + if (cd->status == BCE_COMP_ABORTED) { + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + cnt++; + continue; + } + + msg = &eq->data[sq->head]; + /* + * Route events to appropriate handlers. + * Strip 0x4000 flag; firmware uses it as a + * variant marker. + */ + if (msg->cmd & BCE_VHCI_CMD_REPLY_FLAG) + bce_vhci_cmd_deliver_completion(vhci, msg); + else { + uint16_t base_cmd = msg->cmd & *** 4335 LINES SKIPPED ***
