The branch main has been updated by chuck: URL: https://cgit.FreeBSD.org/src/commit/?id=9f1fa1a46117a418e7f7fbfb1be437100fad531f
commit 9f1fa1a46117a418e7f7fbfb1be437100fad531f Author: Chuck Tuffli <[email protected]> AuthorDate: 2021-12-15 07:16:49 +0000 Commit: Chuck Tuffli <[email protected]> CommitDate: 2021-12-15 07:16:49 +0000 bhyve nvme: Add AEN support to NVMe emulation Add Asynchronous Event Notification infrastructure to the NVMe emulation. Reviewed by: imp, grehan MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D32952 --- usr.sbin/bhyve/pci_nvme.c | 273 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 266 insertions(+), 7 deletions(-) diff --git a/usr.sbin/bhyve/pci_nvme.c b/usr.sbin/bhyve/pci_nvme.c index 034c9409cea3..6fd1aa14b9e3 100644 --- a/usr.sbin/bhyve/pci_nvme.c +++ b/usr.sbin/bhyve/pci_nvme.c @@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$"); #include <assert.h> #include <pthread.h> +#include <pthread_np.h> #include <semaphore.h> #include <stdbool.h> #include <stddef.h> @@ -250,11 +251,39 @@ struct nvme_feature_obj { #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) +typedef enum { + PCI_NVME_AE_TYPE_ERROR = 0, + PCI_NVME_AE_TYPE_SMART, + PCI_NVME_AE_TYPE_NOTICE, + PCI_NVME_AE_TYPE_IO_CMD = 6, + PCI_NVME_AE_TYPE_VENDOR = 7, + PCI_NVME_AE_TYPE_MAX /* Must be last */ +} pci_nvme_async_type; + +/* Asynchronous Event Requests */ struct pci_nvme_aer { STAILQ_ENTRY(pci_nvme_aer) link; uint16_t cid; /* Command ID of the submitted AER */ }; +typedef enum { + PCI_NVME_AE_INFO_NS_ATTR_CHANGED = 0, + PCI_NVME_AE_INFO_FW_ACTIVATION, + PCI_NVME_AE_INFO_TELEMETRY_CHANGE, + PCI_NVME_AE_INFO_ANA_CHANGE, + PCI_NVME_AE_INFO_PREDICT_LATENCY_CHANGE, + PCI_NVME_AE_INFO_LBA_STATUS_ALERT, + PCI_NVME_AE_INFO_ENDURANCE_GROUP_CHANGE, + PCI_NVME_AE_INFO_MAX, +} pci_nvme_async_info; + +/* Asynchronous Event Notifications */ +struct pci_nvme_aen { + pci_nvme_async_type atype; + uint32_t event_data; + bool posted; +}; + struct pci_nvme_softc { struct pci_devinst *nsc_pi; @@ -302,10 +331,21 @@ struct pci_nvme_softc { uint32_t write_dunits_remainder; STAILQ_HEAD(, pci_nvme_aer) aer_list; + pthread_mutex_t aer_mtx; uint32_t aer_count; + struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX]; + pthread_t aen_tid; + pthread_mutex_t aen_mtx; + pthread_cond_t aen_cond; }; +static void pci_nvme_cq_update(struct pci_nvme_softc *sc, + struct nvme_completion_queue *cq, + uint32_t cdw0, + uint16_t cid, + uint16_t sqid, + uint16_t status); static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); static void pci_nvme_io_done(struct blockif_req *, int); @@ -360,6 +400,8 @@ static void nvme_feature_iv_config(struct pci_nvme_softc *, struct nvme_command *, struct nvme_completion *); +static void *aen_thr(void *arg); + static __inline void cpywithpad(char *dst, size_t dst_size, const char *src, char pad) { @@ -623,6 +665,8 @@ pci_nvme_init_features(struct pci_nvme_softc *sc) sc->feat[NVME_FEAT_NUMBER_OF_QUEUES].set = nvme_feature_num_queues; sc->feat[NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION].set = nvme_feature_iv_config; + /* Enable all AENs by default */ + sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 = 0x31f; sc->feat[NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG].get = nvme_feature_invalid_cb; sc->feat[NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW].get = @@ -630,32 +674,42 @@ pci_nvme_init_features(struct pci_nvme_softc *sc) } static void -pci_nvme_aer_init(struct pci_nvme_softc *sc) +pci_nvme_aer_reset(struct pci_nvme_softc *sc) { STAILQ_INIT(&sc->aer_list); sc->aer_count = 0; } +static void +pci_nvme_aer_init(struct pci_nvme_softc *sc) +{ + + pthread_mutex_init(&sc->aer_mtx, NULL); + pci_nvme_aer_reset(sc); +} + static void pci_nvme_aer_destroy(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer = NULL; + pthread_mutex_lock(&sc->aer_mtx); while (!STAILQ_EMPTY(&sc->aer_list)) { aer = STAILQ_FIRST(&sc->aer_list); STAILQ_REMOVE_HEAD(&sc->aer_list, link); free(aer); } + pthread_mutex_unlock(&sc->aer_mtx); - pci_nvme_aer_init(sc); + pci_nvme_aer_reset(sc); } static bool pci_nvme_aer_available(struct pci_nvme_softc *sc) { - return (!STAILQ_EMPTY(&sc->aer_list)); + return (sc->aer_count != 0); } static bool @@ -687,11 +741,13 @@ pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid) if (aer == NULL) return (-1); - sc->aer_count++; - /* Save the Command ID for use in the completion message */ aer->cid = cid; + + pthread_mutex_lock(&sc->aer_mtx); + sc->aer_count++; STAILQ_INSERT_TAIL(&sc->aer_list, aer, link); + pthread_mutex_unlock(&sc->aer_mtx); return (0); } @@ -707,15 +763,208 @@ pci_nvme_aer_get(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer = NULL; + pthread_mutex_lock(&sc->aer_mtx); aer = STAILQ_FIRST(&sc->aer_list); if (aer != NULL) { STAILQ_REMOVE_HEAD(&sc->aer_list, link); sc->aer_count--; } + pthread_mutex_unlock(&sc->aer_mtx); return (aer); } +static void +pci_nvme_aen_reset(struct pci_nvme_softc *sc) +{ + uint32_t atype; + + memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen)); + + for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { + sc->aen[atype].atype = atype; + } +} + +static void +pci_nvme_aen_init(struct pci_nvme_softc *sc) +{ + char nstr[80]; + + pci_nvme_aen_reset(sc); + + pthread_mutex_init(&sc->aen_mtx, NULL); + pthread_create(&sc->aen_tid, NULL, aen_thr, sc); + snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot, + sc->nsc_pi->pi_func); + pthread_set_name_np(sc->aen_tid, nstr); +} + +static void +pci_nvme_aen_destroy(struct pci_nvme_softc *sc) +{ + + pci_nvme_aen_reset(sc); +} + +/* Notify the AEN thread of pending work */ +static void +pci_nvme_aen_notify(struct pci_nvme_softc *sc) +{ + + pthread_cond_signal(&sc->aen_cond); +} + +/* + * Post an Asynchronous Event Notification + */ +static int32_t +pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype, + uint32_t event_data) +{ + struct pci_nvme_aen *aen; + + if (atype >= PCI_NVME_AE_TYPE_MAX) { + return(EINVAL); + } + + pthread_mutex_lock(&sc->aen_mtx); + aen = &sc->aen[atype]; + + /* Has the controller already posted an event of this type? */ + if (aen->posted) { + pthread_mutex_unlock(&sc->aen_mtx); + return(EALREADY); + } + + aen->event_data = event_data; + aen->posted = true; + pthread_mutex_unlock(&sc->aen_mtx); + + pci_nvme_aen_notify(sc); + + return(0); +} + +static void +pci_nvme_aen_process(struct pci_nvme_softc *sc) +{ + struct pci_nvme_aer *aer; + struct pci_nvme_aen *aen; + pci_nvme_async_type atype; + uint32_t mask; + uint16_t status; + uint8_t lid; + + assert(pthread_mutex_isowned_np(&sc->aen_mtx)); + for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { + aen = &sc->aen[atype]; + /* Previous iterations may have depleted the available AER's */ + if (!pci_nvme_aer_available(sc)) { + DPRINTF("%s: no AER", __func__); + break; + } + + if (!aen->posted) { + DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype); + continue; + } + + status = NVME_SC_SUCCESS; + + /* Is the event masked? */ + mask = + sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11; + + DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data); + switch (atype) { + case PCI_NVME_AE_TYPE_ERROR: + lid = NVME_LOG_ERROR; + break; + case PCI_NVME_AE_TYPE_SMART: + mask &= 0xff; + if ((mask & aen->event_data) == 0) + continue; + lid = NVME_LOG_HEALTH_INFORMATION; + break; + case PCI_NVME_AE_TYPE_NOTICE: + if (aen->event_data >= PCI_NVME_AE_INFO_MAX) { + EPRINTLN("%s unknown AEN notice type %u", + __func__, aen->event_data); + status = NVME_SC_INTERNAL_DEVICE_ERROR; + break; + } + mask >>= 8; + if (((1 << aen->event_data) & mask) == 0) + continue; + switch (aen->event_data) { + case PCI_NVME_AE_INFO_NS_ATTR_CHANGED: + lid = NVME_LOG_CHANGED_NAMESPACE; + break; + case PCI_NVME_AE_INFO_FW_ACTIVATION: + lid = NVME_LOG_FIRMWARE_SLOT; + break; + case PCI_NVME_AE_INFO_TELEMETRY_CHANGE: + lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED; + break; + case PCI_NVME_AE_INFO_ANA_CHANGE: + lid = NVME_LOG_ASYMMETRIC_NAMESPAVE_ACCESS; //TODO spelling + break; + case PCI_NVME_AE_INFO_PREDICT_LATENCY_CHANGE: + lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE; + break; + case PCI_NVME_AE_INFO_LBA_STATUS_ALERT: + lid = NVME_LOG_LBA_STATUS_INFORMATION; + break; + case PCI_NVME_AE_INFO_ENDURANCE_GROUP_CHANGE: + lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE; + break; + default: + lid = 0; + } + break; + default: + /* bad type?!? */ + EPRINTLN("%s unknown AEN type %u", __func__, atype); + status = NVME_SC_INTERNAL_DEVICE_ERROR; + break; + } + + aer = pci_nvme_aer_get(sc); + assert(aer != NULL); + + DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype); + pci_nvme_cq_update(sc, &sc->compl_queues[0], + (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */ + aer->cid, + 0, /* SQID */ + status); + + aen->event_data = 0; + aen->posted = false; + + pci_generate_msix(sc->nsc_pi, 0); + } +} + +static void * +aen_thr(void *arg) +{ + struct pci_nvme_softc *sc; + + sc = arg; + + pthread_mutex_lock(&sc->aen_mtx); + for (;;) { + pci_nvme_aen_process(sc); + pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx); + } + pthread_mutex_unlock(&sc->aen_mtx); + + pthread_exit(NULL); + return (NULL); +} + static void pci_nvme_reset_locked(struct pci_nvme_softc *sc) { @@ -756,6 +1005,7 @@ pci_nvme_reset_locked(struct pci_nvme_softc *sc) sc->num_q_is_set = false; pci_nvme_aer_destroy(sc); + pci_nvme_aen_destroy(sc); } static void @@ -1406,8 +1656,13 @@ nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command, if (feat->set) feat->set(sc, feat, command, compl); - if (compl->status == NVME_SC_SUCCESS) + DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11); + if (compl->status == NVME_SC_SUCCESS) { feat->cdw11 = command->cdw11; + if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) && + (command->cdw11 != 0)) + pci_nvme_aen_notify(sc); + } return (0); } @@ -1525,7 +1780,8 @@ static int nvme_opc_async_event_req(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { - DPRINTF("%s async event request 0x%x", __func__, command->cdw11); + DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__, + sc->aer_count, sc->ctrldata.aerl, command->cid); /* Don't exceed the Async Event Request Limit (AERL). */ if (pci_nvme_aer_limit_reached(sc)) { @@ -1546,6 +1802,7 @@ nvme_opc_async_event_req(struct pci_nvme_softc* sc, * there is an event reflective of the request to get event. */ compl->status = NVME_NO_STATUS; + pci_nvme_aen_notify(sc); return (0); } @@ -2025,6 +2282,7 @@ nvme_opc_write_read(struct pci_nvme_softc *sc, lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; nblocks = (cmd->cdw12 & 0xFFFF) + 1; + if (pci_nvme_out_of_range(nvstore, lba, nblocks)) { WPRINTF("%s command would exceed LBA range", __func__); pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); @@ -2789,6 +3047,7 @@ pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl) pci_nvme_init_features(sc); pci_nvme_aer_init(sc); + pci_nvme_aen_init(sc); pci_nvme_reset(sc);
