Author: mav
Date: Thu Feb 21 00:44:26 2019
New Revision: 344401
URL: https://svnweb.freebsd.org/changeset/base/344401
Log:
MFC r302669,302677-302686,303761,304602,304603,305027-305028,305259,
305710,305711,308067-308070,308178,308179,308230,308553,309338,309526,
343125 (by cem): Synchronize ioat(4) with head.
Most of these changes are 3 years old, just never got merged.
Modified:
stable/11/sys/dev/ioat/ioat.c
stable/11/sys/dev/ioat/ioat.h
stable/11/sys/dev/ioat/ioat_hw.h
stable/11/sys/dev/ioat/ioat_internal.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/dev/ioat/ioat.c
==============================================================================
--- stable/11/sys/dev/ioat/ioat.c Thu Feb 21 00:17:24 2019
(r344400)
+++ stable/11/sys/dev/ioat/ioat.c Thu Feb 21 00:44:26 2019
(r344401)
@@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/conf.h>
+#include <sys/fail.h>
#include <sys/ioccom.h>
#include <sys/kernel.h>
#include <sys/lock.h>
@@ -62,7 +63,6 @@ __FBSDID("$FreeBSD$");
#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL
#endif
#define IOAT_REFLK (&ioat->submit_lock)
-#define IOAT_SHRINK_PERIOD (10 * hz)
static int ioat_probe(device_t device);
static int ioat_attach(device_t device);
@@ -81,23 +81,14 @@ static void ioat_process_events(struct ioat_softc *ioa
static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
static void ioat_free_ring(struct ioat_softc *, uint32_t size,
- struct ioat_descriptor **);
-static void ioat_free_ring_entry(struct ioat_softc *ioat,
- struct ioat_descriptor *desc);
-static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *,
- int mflags);
+ struct ioat_descriptor *);
static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags);
-static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat,
+static union ioat_hw_descriptor *ioat_get_descriptor(struct ioat_softc *,
uint32_t index);
-static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *,
- uint32_t size, boolean_t need_dscr, int mflags);
-static int ring_grow(struct ioat_softc *, uint32_t oldorder,
- struct ioat_descriptor **);
-static int ring_shrink(struct ioat_softc *, uint32_t oldorder,
- struct ioat_descriptor **);
+static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *,
+ uint32_t index);
static void ioat_halted_debug(struct ioat_softc *, uint32_t);
static void ioat_poll_timer_callback(void *arg);
-static void ioat_shrink_timer_callback(void *arg);
static void dump_descriptor(void *hw_desc);
static void ioat_submit_single(struct ioat_softc *ioat);
static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
@@ -134,6 +125,10 @@ int g_ioat_debug_level = 0;
SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
+unsigned g_ioat_ring_order = 13;
+SYSCTL_UINT(_hw_ioat, OID_AUTO, ring_order, CTLFLAG_RDTUN, &g_ioat_ring_order,
+ 0, "Set IOAT ring order. (1 << this) == ring size.");
+
/*
* OS <-> Driver interface structures
*/
@@ -335,7 +330,6 @@ ioat_detach(device_t device)
ioat_teardown_intr(ioat);
callout_drain(&ioat->poll_timer);
- callout_drain(&ioat->shrink_timer);
pci_disable_busmaster(device);
@@ -353,7 +347,12 @@ ioat_detach(device_t device)
bus_dma_tag_destroy(ioat->comp_update_tag);
}
- bus_dma_tag_destroy(ioat->hw_desc_tag);
+ if (ioat->hw_desc_ring != NULL) {
+ bus_dmamap_unload(ioat->hw_desc_tag, ioat->hw_desc_map);
+ bus_dmamem_free(ioat->hw_desc_tag, ioat->hw_desc_ring,
+ ioat->hw_desc_map);
+ bus_dma_tag_destroy(ioat->hw_desc_tag);
+ }
return (0);
}
@@ -387,8 +386,8 @@ ioat_start_channel(struct ioat_softc *ioat)
/* Submit 'NULL' operation manually to avoid quiescing flag */
desc = ioat_get_ring_entry(ioat, ioat->head);
+ hw_desc = &ioat_get_descriptor(ioat, ioat->head)->dma;
dmadesc = &desc->bus_dmadesc;
- hw_desc = desc->u.dma;
dmadesc->callback_fn = NULL;
dmadesc->callback_arg = NULL;
@@ -425,9 +424,10 @@ static int
ioat3_attach(device_t device)
{
struct ioat_softc *ioat;
- struct ioat_descriptor **ring;
- struct ioat_descriptor *next;
+ struct ioat_descriptor *ring;
struct ioat_dma_hw_descriptor *dma_hw_desc;
+ void *hw_desc;
+ size_t ringsz;
int i, num_descriptors;
int error;
uint8_t xfercap;
@@ -452,7 +452,6 @@ ioat3_attach(device_t device)
mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF);
callout_init(&ioat->poll_timer, 1);
- callout_init(&ioat->shrink_timer, 1);
TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat);
/* Establish lock order for Witness */
@@ -461,7 +460,7 @@ ioat3_attach(device_t device)
mtx_unlock(&ioat->cleanup_lock);
mtx_unlock(&ioat->submit_lock);
- ioat->is_resize_pending = FALSE;
+ ioat->is_submitter_processing = FALSE;
ioat->is_completion_pending = FALSE;
ioat->is_reset_pending = FALSE;
ioat->is_channel_running = FALSE;
@@ -482,37 +481,43 @@ ioat3_attach(device_t device)
if (error != 0)
return (error);
- ioat->ring_size_order = IOAT_MIN_ORDER;
-
+ ioat->ring_size_order = g_ioat_ring_order;
num_descriptors = 1 << ioat->ring_size_order;
+ ringsz = sizeof(struct ioat_dma_hw_descriptor) * num_descriptors;
- bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0,
- BUS_SPACE_MAXADDR_40BIT, BUS_SPACE_MAXADDR, NULL, NULL,
- sizeof(struct ioat_dma_hw_descriptor), 1,
- sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL,
+ error = bus_dma_tag_create(bus_get_dma_tag(ioat->device),
+ 2 * 1024 * 1024, 0x0, (bus_addr_t)BUS_SPACE_MAXADDR_40BIT,
+ BUS_SPACE_MAXADDR, NULL, NULL, ringsz, 1, ringsz, 0, NULL, NULL,
&ioat->hw_desc_tag);
+ if (error != 0)
+ return (error);
+ error = bus_dmamem_alloc(ioat->hw_desc_tag, &hw_desc,
+ BUS_DMA_ZERO | BUS_DMA_WAITOK, &ioat->hw_desc_map);
+ if (error != 0)
+ return (error);
+
+ error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
+ ringsz, ioat_dmamap_cb, &ioat->hw_desc_bus_addr, BUS_DMA_WAITOK);
+ if (error)
+ return (error);
+
+ ioat->hw_desc_ring = hw_desc;
+
ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
M_ZERO | M_WAITOK);
ring = ioat->ring;
for (i = 0; i < num_descriptors; i++) {
- ring[i] = ioat_alloc_ring_entry(ioat, M_WAITOK);
- if (ring[i] == NULL)
- return (ENOMEM);
-
- ring[i]->id = i;
+ memset(&ring[i].bus_dmadesc, 0, sizeof(ring[i].bus_dmadesc));
+ ring[i].id = i;
}
- for (i = 0; i < num_descriptors - 1; i++) {
- next = ring[i + 1];
- dma_hw_desc = ring[i]->u.dma;
-
- dma_hw_desc->next = next->hw_desc_bus_addr;
+ for (i = 0; i < num_descriptors; i++) {
+ dma_hw_desc = &ioat->hw_desc_ring[i].dma;
+ dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1);
}
- ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
-
ioat->head = ioat->hw_head = 0;
ioat->tail = 0;
ioat->last_seen = 0;
@@ -662,8 +667,6 @@ ioat_process_events(struct ioat_softc *ioat)
boolean_t pending;
int error;
- CTR0(KTR_IOAT, __func__);
-
mtx_lock(&ioat->cleanup_lock);
/*
@@ -680,31 +683,42 @@ ioat_process_events(struct ioat_softc *ioat)
comp_update = *ioat->comp_update;
status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
+ if (status < ioat->hw_desc_bus_addr ||
+ status >= ioat->hw_desc_bus_addr + (1 << ioat->ring_size_order) *
+ sizeof(struct ioat_generic_hw_descriptor))
+ panic("Bogus completion address %jx (channel %u)",
+ (uintmax_t)status, ioat->chan_idx);
+
if (status == ioat->last_seen) {
/*
* If we landed in process_events and nothing has been
* completed, check for a timeout due to channel halt.
*/
- comp_update = ioat_get_chansts(ioat);
goto out;
}
+ CTR4(KTR_IOAT, "%s channel=%u hw_status=0x%lx last_seen=0x%lx",
+ __func__, ioat->chan_idx, comp_update, ioat->last_seen);
- while (1) {
+ while (RING_PHYS_ADDR(ioat, ioat->tail - 1) != status) {
desc = ioat_get_ring_entry(ioat, ioat->tail);
dmadesc = &desc->bus_dmadesc;
- CTR1(KTR_IOAT, "completing desc %d", ioat->tail);
+ CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) ok cb
%p(%p)",
+ ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn,
+ dmadesc->callback_arg);
if (dmadesc->callback_fn != NULL)
dmadesc->callback_fn(dmadesc->callback_arg, 0);
completed++;
ioat->tail++;
- if (desc->hw_desc_bus_addr == status)
- break;
}
+ CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__,
+ ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat));
- ioat->last_seen = desc->hw_desc_bus_addr;
- ioat->stats.descriptors_processed += completed;
+ if (completed != 0) {
+ ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1);
+ ioat->stats.descriptors_processed += completed;
+ }
out:
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
@@ -719,8 +733,6 @@ out:
pending = (ioat_get_active(ioat) != 0);
if (!pending && ioat->is_completion_pending) {
ioat->is_completion_pending = FALSE;
- callout_reset(&ioat->shrink_timer, IOAT_SHRINK_PERIOD,
- ioat_shrink_timer_callback, ioat);
callout_stop(&ioat->poll_timer);
}
mtx_unlock(&ioat->submit_lock);
@@ -736,6 +748,12 @@ out:
wakeup(&ioat->tail);
}
+ /*
+ * The device doesn't seem to reliably push suspend/halt statuses to
+ * the channel completion memory address, so poll the device register
+ * here.
+ */
+ comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS;
if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update))
return;
@@ -745,19 +763,30 @@ out:
* Fatal programming error on this DMA channel. Flush any outstanding
* work with error status and restart the engine.
*/
- ioat_log_message(0, "Channel halted due to fatal programming error\n");
mtx_lock(&ioat->submit_lock);
mtx_lock(&ioat->cleanup_lock);
ioat->quiescing = TRUE;
+ /*
+ * This is safe to do here because we have both locks and the submit
+ * queue is quiesced. We know that we will drain all outstanding
+ * events, so ioat_reset_hw can't deadlock. It is necessary to
+ * protect other ioat_process_event threads from racing ioat_reset_hw,
+ * reading an indeterminate hw state, and attempting to continue
+ * issuing completions.
+ */
+ ioat->resetting_cleanup = TRUE;
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
- ioat_halted_debug(ioat, chanerr);
+ if (1 <= g_ioat_debug_level)
+ ioat_halted_debug(ioat, chanerr);
ioat->stats.last_halt_chanerr = chanerr;
while (ioat_get_active(ioat) > 0) {
desc = ioat_get_ring_entry(ioat, ioat->tail);
dmadesc = &desc->bus_dmadesc;
- CTR1(KTR_IOAT, "completing err desc %d", ioat->tail);
+ CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) err cb
%p(%p)",
+ ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn,
+ dmadesc->callback_arg);
if (dmadesc->callback_fn != NULL)
dmadesc->callback_fn(dmadesc->callback_arg,
@@ -768,7 +797,14 @@ out:
ioat->stats.descriptors_processed++;
ioat->stats.descriptors_error++;
}
+ CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__,
+ ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat));
+ if (ioat->is_completion_pending) {
+ ioat->is_completion_pending = FALSE;
+ callout_stop(&ioat->poll_timer);
+ }
+
/* Clear error status */
ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
@@ -869,6 +905,15 @@ ioat_get_max_io_size(bus_dmaengine_t dmaengine)
return (ioat->max_xfer_size);
}
+uint32_t
+ioat_get_capabilities(bus_dmaengine_t dmaengine)
+{
+ struct ioat_softc *ioat;
+
+ ioat = to_ioat_softc(dmaengine);
+ return (ioat->capabilities);
+}
+
int
ioat_set_interrupt_coalesce(bus_dmaengine_t dmaengine, uint16_t delay)
{
@@ -902,7 +947,8 @@ ioat_acquire(bus_dmaengine_t dmaengine)
ioat = to_ioat_softc(dmaengine);
mtx_lock(&ioat->submit_lock);
- CTR0(KTR_IOAT, __func__);
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
+ ioat->acq_head = ioat->head;
}
int
@@ -926,8 +972,22 @@ ioat_release(bus_dmaengine_t dmaengine)
struct ioat_softc *ioat;
ioat = to_ioat_softc(dmaengine);
- CTR0(KTR_IOAT, __func__);
- ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->hw_head);
+ CTR4(KTR_IOAT, "%s channel=%u dispatch1 hw_head=%u head=%u", __func__,
+ ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head);
+ KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */);
+ CTR4(KTR_IOAT, "%s channel=%u dispatch2 hw_head=%u head=%u", __func__,
+ ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head);
+
+ if (ioat->acq_head != ioat->head) {
+ ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET,
+ (uint16_t)ioat->hw_head);
+
+ if (!ioat->is_completion_pending) {
+ ioat->is_completion_pending = TRUE;
+ callout_reset(&ioat->poll_timer, 1,
+ ioat_poll_timer_callback, ioat);
+ }
+ }
mtx_unlock(&ioat->submit_lock);
}
@@ -960,7 +1020,7 @@ ioat_op_generic(struct ioat_softc *ioat, uint8_t op,
return (NULL);
desc = ioat_get_ring_entry(ioat, ioat->head);
- hw_desc = desc->u.generic;
+ hw_desc = &ioat_get_descriptor(ioat, ioat->head)->generic;
hw_desc->u.control_raw = 0;
hw_desc->u.control_generic.op = op;
@@ -988,15 +1048,15 @@ ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_cal
struct ioat_descriptor *desc;
struct ioat_softc *ioat;
- CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn,
callback_arg, flags);
if (desc == NULL)
return (NULL);
- hw_desc = desc->u.dma;
+ hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma;
hw_desc->u.control.null = 1;
ioat_submit_single(ioat);
return (&desc->bus_dmadesc);
@@ -1011,7 +1071,6 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
struct ioat_descriptor *desc;
struct ioat_softc *ioat;
- CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
if (((src | dst) & (0xffffull << 48)) != 0) {
@@ -1025,11 +1084,13 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
if (desc == NULL)
return (NULL);
- hw_desc = desc->u.dma;
+ hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma;
if (g_ioat_debug_level >= 3)
dump_descriptor(hw_desc);
ioat_submit_single(ioat);
+ CTR6(KTR_IOAT, "%s channel=%u desc=%p dest=%lx src=%lx len=%lx",
+ __func__, ioat->chan_idx, &desc->bus_dmadesc, dst, src, len);
return (&desc->bus_dmadesc);
}
@@ -1042,8 +1103,8 @@ ioat_copy_8k_aligned(bus_dmaengine_t dmaengine, bus_ad
struct ioat_descriptor *desc;
struct ioat_softc *ioat;
- CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
if (((src1 | src2 | dst1 | dst2) & (0xffffull << 48)) != 0) {
ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n",
@@ -1061,7 +1122,7 @@ ioat_copy_8k_aligned(bus_dmaengine_t dmaengine, bus_ad
if (desc == NULL)
return (NULL);
- hw_desc = desc->u.dma;
+ hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma;
if (src2 != src1 + PAGE_SIZE) {
hw_desc->u.control.src_page_break = 1;
hw_desc->next_src_addr = src2;
@@ -1089,8 +1150,8 @@ ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t ds
uint32_t teststore;
uint8_t op;
- CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
if ((ioat->capabilities & IOAT_DMACAP_MOVECRC) == 0) {
ioat_log_message(0, "%s: Device lacks MOVECRC capability\n",
@@ -1138,7 +1199,7 @@ ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t ds
if (desc == NULL)
return (NULL);
- hw_desc = desc->u.crc32;
+ hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32;
if ((flags & DMA_CRC_INLINE) == 0)
hw_desc->crc_address = crcptr;
@@ -1168,8 +1229,8 @@ ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bu
uint32_t teststore;
uint8_t op;
- CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
if ((ioat->capabilities & IOAT_DMACAP_CRC) == 0) {
ioat_log_message(0, "%s: Device lacks CRC capability\n",
@@ -1217,7 +1278,7 @@ ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bu
if (desc == NULL)
return (NULL);
- hw_desc = desc->u.crc32;
+ hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32;
if ((flags & DMA_CRC_INLINE) == 0)
hw_desc->crc_address = crcptr;
@@ -1245,8 +1306,8 @@ ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t d
struct ioat_descriptor *desc;
struct ioat_softc *ioat;
- CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
if ((ioat->capabilities & IOAT_DMACAP_BFILL) == 0) {
ioat_log_message(0, "%s: Device lacks BFILL capability\n",
@@ -1265,7 +1326,7 @@ ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t d
if (desc == NULL)
return (NULL);
- hw_desc = desc->u.fill;
+ hw_desc = &ioat_get_descriptor(ioat, desc->id)->fill;
if (g_ioat_debug_level >= 3)
dump_descriptor(hw_desc);
@@ -1290,60 +1351,6 @@ ioat_get_ring_space(struct ioat_softc *ioat)
return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1);
}
-static struct ioat_descriptor *
-ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
-{
- struct ioat_generic_hw_descriptor *hw_desc;
- struct ioat_descriptor *desc;
- int error, busdmaflag;
-
- error = ENOMEM;
- hw_desc = NULL;
-
- if ((mflags & M_WAITOK) != 0)
- busdmaflag = BUS_DMA_WAITOK;
- else
- busdmaflag = BUS_DMA_NOWAIT;
-
- desc = malloc(sizeof(*desc), M_IOAT, mflags);
- if (desc == NULL)
- goto out;
-
- bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc,
- BUS_DMA_ZERO | busdmaflag, &ioat->hw_desc_map);
- if (hw_desc == NULL)
- goto out;
-
- memset(&desc->bus_dmadesc, 0, sizeof(desc->bus_dmadesc));
- desc->u.generic = hw_desc;
-
- error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
- sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr,
- busdmaflag);
- if (error)
- goto out;
-
-out:
- if (error) {
- ioat_free_ring_entry(ioat, desc);
- return (NULL);
- }
- return (desc);
-}
-
-static void
-ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
-{
-
- if (desc == NULL)
- return;
-
- if (desc->u.generic)
- bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic,
- ioat->hw_desc_map);
- free(desc, M_IOAT);
-}
-
/*
* Reserves space in this IOAT descriptor ring by ensuring enough slots remain
* for 'num_descs'.
@@ -1363,114 +1370,70 @@ ioat_free_ring_entry(struct ioat_softc *ioat, struct i
static int
ioat_reserve_space(struct ioat_softc *ioat, uint32_t num_descs, int mflags)
{
- struct ioat_descriptor **new_ring;
- uint32_t order;
+ boolean_t dug;
int error;
mtx_assert(&ioat->submit_lock, MA_OWNED);
error = 0;
+ dug = FALSE;
- if (num_descs < 1 || num_descs > (1 << IOAT_MAX_ORDER)) {
+ if (num_descs < 1 || num_descs >= (1 << ioat->ring_size_order)) {
error = EINVAL;
goto out;
}
- if (ioat->quiescing) {
- error = ENXIO;
- goto out;
- }
for (;;) {
+ if (ioat->quiescing) {
+ error = ENXIO;
+ goto out;
+ }
+
if (ioat_get_ring_space(ioat) >= num_descs)
goto out;
- order = ioat->ring_size_order;
- if (ioat->is_resize_pending || order == IOAT_MAX_ORDER) {
- if ((mflags & M_WAITOK) != 0) {
- msleep(&ioat->tail, &ioat->submit_lock, 0,
- "ioat_rsz", 0);
- continue;
- }
+ CTR3(KTR_IOAT, "%s channel=%u starved (%u)", __func__,
+ ioat->chan_idx, num_descs);
- error = EAGAIN;
- break;
- }
-
- ioat->is_resize_pending = TRUE;
- for (;;) {
+ if (!dug && !ioat->is_submitter_processing) {
+ ioat->is_submitter_processing = TRUE;
mtx_unlock(&ioat->submit_lock);
- new_ring = ioat_prealloc_ring(ioat, 1 << (order + 1),
- TRUE, mflags);
+ CTR2(KTR_IOAT, "%s channel=%u attempting to process
events",
+ __func__, ioat->chan_idx);
+ ioat_process_events(ioat);
mtx_lock(&ioat->submit_lock);
- KASSERT(ioat->ring_size_order == order,
- ("is_resize_pending should protect order"));
-
- if (new_ring == NULL) {
- KASSERT((mflags & M_WAITOK) == 0,
- ("allocation failed"));
- error = EAGAIN;
- break;
- }
-
- error = ring_grow(ioat, order, new_ring);
- if (error == 0)
- break;
+ dug = TRUE;
+ KASSERT(ioat->is_submitter_processing == TRUE,
+ ("is_submitter_processing"));
+ ioat->is_submitter_processing = FALSE;
+ wakeup(&ioat->tail);
+ continue;
}
- ioat->is_resize_pending = FALSE;
- wakeup(&ioat->tail);
- if (error)
+
+ if ((mflags & M_WAITOK) == 0) {
+ error = EAGAIN;
break;
+ }
+ CTR2(KTR_IOAT, "%s channel=%u blocking on completions",
+ __func__, ioat->chan_idx);
+ msleep(&ioat->tail, &ioat->submit_lock, 0,
+ "ioat_full", 0);
+ continue;
}
out:
mtx_assert(&ioat->submit_lock, MA_OWNED);
+ KASSERT(!ioat->quiescing || error == ENXIO,
+ ("reserved during quiesce"));
return (error);
}
-static struct ioat_descriptor **
-ioat_prealloc_ring(struct ioat_softc *ioat, uint32_t size, boolean_t need_dscr,
- int mflags)
-{
- struct ioat_descriptor **ring;
- uint32_t i;
- int error;
-
- KASSERT(size > 0 && powerof2(size), ("bogus size"));
-
- ring = malloc(size * sizeof(*ring), M_IOAT, M_ZERO | mflags);
- if (ring == NULL)
- return (NULL);
-
- if (need_dscr) {
- error = ENOMEM;
- for (i = size / 2; i < size; i++) {
- ring[i] = ioat_alloc_ring_entry(ioat, mflags);
- if (ring[i] == NULL)
- goto out;
- ring[i]->id = i;
- }
- }
- error = 0;
-
-out:
- if (error != 0 && ring != NULL) {
- ioat_free_ring(ioat, size, ring);
- ring = NULL;
- }
- return (ring);
-}
-
static void
ioat_free_ring(struct ioat_softc *ioat, uint32_t size,
- struct ioat_descriptor **ring)
+ struct ioat_descriptor *ring)
{
- uint32_t i;
- for (i = 0; i < size; i++) {
- if (ring[i] != NULL)
- ioat_free_ring_entry(ioat, ring[i]);
- }
free(ring, M_IOAT);
}
@@ -1478,164 +1441,20 @@ static struct ioat_descriptor *
ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index)
{
- return (ioat->ring[index % (1 << ioat->ring_size_order)]);
+ return (&ioat->ring[index % (1 << ioat->ring_size_order)]);
}
-static int
-ring_grow(struct ioat_softc *ioat, uint32_t oldorder,
- struct ioat_descriptor **newring)
+static union ioat_hw_descriptor *
+ioat_get_descriptor(struct ioat_softc *ioat, uint32_t index)
{
- struct ioat_descriptor *tmp, *next;
- struct ioat_dma_hw_descriptor *hw;
- uint32_t oldsize, newsize, head, tail, i, end;
- int error;
- CTR0(KTR_IOAT, __func__);
-
- mtx_assert(&ioat->submit_lock, MA_OWNED);
-
- if (oldorder != ioat->ring_size_order || oldorder >= IOAT_MAX_ORDER) {
- error = EINVAL;
- goto out;
- }
-
- oldsize = (1 << oldorder);
- newsize = (1 << (oldorder + 1));
-
- mtx_lock(&ioat->cleanup_lock);
-
- head = ioat->head & (oldsize - 1);
- tail = ioat->tail & (oldsize - 1);
-
- /* Copy old descriptors to new ring */
- for (i = 0; i < oldsize; i++)
- newring[i] = ioat->ring[i];
-
- /*
- * If head has wrapped but tail hasn't, we must swap some descriptors
- * around so that tail can increment directly to head.
- */
- if (head < tail) {
- for (i = 0; i <= head; i++) {
- tmp = newring[oldsize + i];
-
- newring[oldsize + i] = newring[i];
- newring[oldsize + i]->id = oldsize + i;
-
- newring[i] = tmp;
- newring[i]->id = i;
- }
- head += oldsize;
- }
-
- KASSERT(head >= tail, ("invariants"));
-
- /* Head didn't wrap; we only need to link in oldsize..newsize */
- if (head < oldsize) {
- i = oldsize - 1;
- end = newsize;
- } else {
- /* Head did wrap; link newhead..newsize and 0..oldhead */
- i = head;
- end = newsize + (head - oldsize) + 1;
- }
-
- /*
- * Fix up hardware ring, being careful not to trample the active
- * section (tail -> head).
- */
- for (; i < end; i++) {
- KASSERT((i & (newsize - 1)) < tail ||
- (i & (newsize - 1)) >= head, ("trampling snake"));
-
- next = newring[(i + 1) & (newsize - 1)];
- hw = newring[i & (newsize - 1)]->u.dma;
- hw->next = next->hw_desc_bus_addr;
- }
-
- free(ioat->ring, M_IOAT);
- ioat->ring = newring;
- ioat->ring_size_order = oldorder + 1;
- ioat->tail = tail;
- ioat->head = head;
- error = 0;
-
- mtx_unlock(&ioat->cleanup_lock);
-out:
- if (error)
- ioat_free_ring(ioat, (1 << (oldorder + 1)), newring);
- return (error);
+ return (&ioat->hw_desc_ring[index % (1 << ioat->ring_size_order)]);
}
-static int
-ring_shrink(struct ioat_softc *ioat, uint32_t oldorder,
- struct ioat_descriptor **newring)
-{
- struct ioat_dma_hw_descriptor *hw;
- struct ioat_descriptor *ent, *next;
- uint32_t oldsize, newsize, current_idx, new_idx, i;
- int error;
-
- CTR0(KTR_IOAT, __func__);
-
- mtx_assert(&ioat->submit_lock, MA_OWNED);
-
- if (oldorder != ioat->ring_size_order || oldorder <= IOAT_MIN_ORDER) {
- error = EINVAL;
- goto out_unlocked;
- }
-
- oldsize = (1 << oldorder);
- newsize = (1 << (oldorder - 1));
-
- mtx_lock(&ioat->cleanup_lock);
-
- /* Can't shrink below current active set! */
- if (ioat_get_active(ioat) >= newsize) {
- error = ENOMEM;
- goto out;
- }
-
- /*
- * Copy current descriptors to the new ring, dropping the removed
- * descriptors.
- */
- for (i = 0; i < newsize; i++) {
- current_idx = (ioat->tail + i) & (oldsize - 1);
- new_idx = (ioat->tail + i) & (newsize - 1);
-
- newring[new_idx] = ioat->ring[current_idx];
- newring[new_idx]->id = new_idx;
- }
-
- /* Free deleted descriptors */
- for (i = newsize; i < oldsize; i++) {
- ent = ioat_get_ring_entry(ioat, ioat->tail + i);
- ioat_free_ring_entry(ioat, ent);
- }
-
- /* Fix up hardware ring. */
- hw = newring[(ioat->tail + newsize - 1) & (newsize - 1)]->u.dma;
- next = newring[(ioat->tail + newsize) & (newsize - 1)];
- hw->next = next->hw_desc_bus_addr;
-
- free(ioat->ring, M_IOAT);
- ioat->ring = newring;
- ioat->ring_size_order = oldorder - 1;
- error = 0;
-
-out:
- mtx_unlock(&ioat->cleanup_lock);
-out_unlocked:
- if (error)
- ioat_free_ring(ioat, (1 << (oldorder - 1)), newring);
- return (error);
-}
-
static void
ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr)
{
- struct ioat_descriptor *desc;
+ union ioat_hw_descriptor *desc;
ioat_log_message(0, "Channel halted (%b)\n", (int)chanerr,
IOAT_CHANERR_STR);
@@ -1644,11 +1463,11 @@ ioat_halted_debug(struct ioat_softc *ioat, uint32_t ch
mtx_assert(&ioat->cleanup_lock, MA_OWNED);
- desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
- dump_descriptor(desc->u.raw);
+ desc = ioat_get_descriptor(ioat, ioat->tail + 0);
+ dump_descriptor(desc);
- desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
- dump_descriptor(desc->u.raw);
+ desc = ioat_get_descriptor(ioat, ioat->tail + 1);
+ dump_descriptor(desc);
}
static void
@@ -1662,52 +1481,6 @@ ioat_poll_timer_callback(void *arg)
ioat_process_events(ioat);
}
-static void
-ioat_shrink_timer_callback(void *arg)
-{
- struct ioat_descriptor **newring;
- struct ioat_softc *ioat;
- uint32_t order;
-
- ioat = arg;
- ioat_log_message(1, "%s\n", __func__);
-
- /* Slowly scale the ring down if idle. */
- mtx_lock(&ioat->submit_lock);
-
- /* Don't run while the hardware is being reset. */
- if (ioat->resetting) {
- mtx_unlock(&ioat->submit_lock);
- return;
- }
-
- order = ioat->ring_size_order;
- if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) {
- mtx_unlock(&ioat->submit_lock);
- goto out;
- }
- ioat->is_resize_pending = TRUE;
- mtx_unlock(&ioat->submit_lock);
-
- newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE,
- M_NOWAIT);
-
- mtx_lock(&ioat->submit_lock);
- KASSERT(ioat->ring_size_order == order,
- ("resize_pending protects order"));
-
- if (newring != NULL)
- ring_shrink(ioat, order, newring);
-
- ioat->is_resize_pending = FALSE;
- mtx_unlock(&ioat->submit_lock);
-
-out:
- if (ioat->ring_size_order > IOAT_MIN_ORDER)
- callout_reset(&ioat->poll_timer, IOAT_SHRINK_PERIOD,
- ioat_shrink_timer_callback, ioat);
-}
-
/*
* Support Functions
*/
@@ -1715,17 +1488,15 @@ static void
ioat_submit_single(struct ioat_softc *ioat)
{
+ mtx_assert(&ioat->submit_lock, MA_OWNED);
+
ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
atomic_add_rel_int(&ioat->head, 1);
atomic_add_rel_int(&ioat->hw_head, 1);
+ CTR5(KTR_IOAT, "%s channel=%u head=%u hw_head=%u tail=%u", __func__,
+ ioat->chan_idx, ioat->head, ioat->hw_head & UINT16_MAX,
+ ioat->tail);
- if (!ioat->is_completion_pending) {
- ioat->is_completion_pending = TRUE;
- callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
- ioat);
- callout_stop(&ioat->shrink_timer);
- }
-
ioat->stats.descriptors_submitted++;
}
@@ -1737,6 +1508,8 @@ ioat_reset_hw(struct ioat_softc *ioat)
unsigned timeout;
int error;
+ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);
+
mtx_lock(IOAT_REFLK);
while (ioat->resetting && !ioat->destroying)
msleep(&ioat->resetting, IOAT_REFLK, 0, "IRH_drain", 0);
@@ -1758,6 +1531,9 @@ ioat_reset_hw(struct ioat_softc *ioat)
ioat->resetting_cleanup = TRUE;
mtx_unlock(&ioat->cleanup_lock);
+ CTR2(KTR_IOAT, "%s channel=%u quiesced and drained", __func__,
+ ioat->chan_idx);
+
status = ioat_get_chansts(ioat);
if (is_ioat_active(status) || is_ioat_idle(status))
ioat_suspend(ioat);
@@ -1778,6 +1554,9 @@ ioat_reset_hw(struct ioat_softc *ioat)
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
+ CTR2(KTR_IOAT, "%s channel=%u hardware suspended", __func__,
+ ioat->chan_idx);
+
/*
* IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors
* that can cause stability issues for IOAT v3.
@@ -1797,6 +1576,8 @@ ioat_reset_hw(struct ioat_softc *ioat)
}
ioat_reset(ioat);
+ CTR2(KTR_IOAT, "%s channel=%u hardware reset", __func__,
+ ioat->chan_idx);
/* Wait at most 20 ms */
for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
@@ -1840,26 +1621,30 @@ ioat_reset_hw(struct ioat_softc *ioat)
ioat->tail = ioat->head = ioat->hw_head = 0;
ioat->last_seen = 0;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
