Module Name: src
Committed By: jmcneill
Date: Mon Feb 19 19:00:42 UTC 2018
Modified Files:
src/sys/dev/ic: pl181.c pl181var.h
Log Message:
Rewrite data transfer path to take advantage of the PL181's 64-byte FIFO.
Before: 134217728 bytes transferred in 43.683 secs (3072539 bytes/sec)
After: 134217728 bytes transferred in 23.789 secs (5642007 bytes/sec)
To generate a diff of this commit:
cvs rdiff -u -r1.4 -r1.5 src/sys/dev/ic/pl181.c
cvs rdiff -u -r1.2 -r1.3 src/sys/dev/ic/pl181var.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/ic/pl181.c
diff -u src/sys/dev/ic/pl181.c:1.4 src/sys/dev/ic/pl181.c:1.5
--- src/sys/dev/ic/pl181.c:1.4 Sun Jun 4 15:08:30 2017
+++ src/sys/dev/ic/pl181.c Mon Feb 19 19:00:42 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: pl181.c,v 1.4 2017/06/04 15:08:30 jmcneill Exp $ */
+/* $NetBSD: pl181.c,v 1.5 2018/02/19 19:00:42 jmcneill Exp $ */
/*-
* Copyright (c) 2015 Jared D. McNeill <jmcne...@invisible.ca>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pl181.c,v 1.4 2017/06/04 15:08:30 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pl181.c,v 1.5 2018/02/19 19:00:42 jmcneill Exp $");
#include <sys/param.h>
#include <sys/bus.h>
@@ -49,6 +49,22 @@ __KERNEL_RCSID(0, "$NetBSD: pl181.c,v 1.
*/
#define PLMMC_MAXXFER rounddown(65535, SDMMC_SECTOR_SIZE)
+/*
+ * PL181 FIFO is 16 words deep (64 bytes)
+ */
+#define PL181_FIFO_DEPTH 64
+
+/*
+ * Data transfer IRQ status bits
+ */
+#define PLMMC_INT_DATA_MASK \
+ (MMCI_INT_DATA_TIMEOUT|MMCI_INT_DATA_CRC_FAIL| \
+ MMCI_INT_TX_FIFO_EMPTY|MMCI_INT_TX_FIFO_HALF_EMPTY| \
+ MMCI_INT_RX_FIFO_FULL|MMCI_INT_RX_FIFO_HALF_FULL| \
+ MMCI_INT_DATA_END|MMCI_INT_DATA_BLOCK_END)
+#define PLMMC_INT_CMD_MASK \
+ (MMCI_INT_CMD_TIMEOUT|MMCI_INT_CMD_RESP_END)
+
static int plmmc_host_reset(sdmmc_chipset_handle_t);
static uint32_t plmmc_host_ocr(sdmmc_chipset_handle_t);
static int plmmc_host_maxblklen(sdmmc_chipset_handle_t);
@@ -63,9 +79,7 @@ static void plmmc_exec_command(sdmmc_chi
static void plmmc_card_enable_intr(sdmmc_chipset_handle_t, int);
static void plmmc_card_intr_ack(sdmmc_chipset_handle_t);
-static int plmmc_wait_status(struct plmmc_softc *, uint32_t, int);
-static int plmmc_pio_wait(struct plmmc_softc *,
- struct sdmmc_command *);
+static int plmmc_wait_cmd(struct plmmc_softc *);
static int plmmc_pio_transfer(struct plmmc_softc *,
struct sdmmc_command *, int);
@@ -86,15 +100,19 @@ static struct sdmmc_chip_functions plmmc
#define MMCI_WRITE(sc, reg, val) \
bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
+#define MMCI_WRITE_MULTI(sc, reg, datap, cnt) \
+ bus_space_write_multi_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (datap), (cnt))
#define MMCI_READ(sc, reg) \
bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg))
+#define MMCI_READ_MULTI(sc, reg, datap, cnt) \
+ bus_space_read_multi_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (datap), (cnt))
void
plmmc_init(struct plmmc_softc *sc)
{
struct sdmmcbus_attach_args saa;
- mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_BIO);
+ mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
cv_init(&sc->sc_intr_cv, "plmmcirq");
#ifdef PLMMC_DEBUG
@@ -130,80 +148,124 @@ plmmc_init(struct plmmc_softc *sc)
sc->sc_sdmmc_dev = config_found(sc->sc_dev, &saa, NULL);
}
-int
-plmmc_intr(void *priv)
+static int
+plmmc_intr_xfer(struct plmmc_softc *sc, struct sdmmc_command *cmd)
{
- struct plmmc_softc *sc = priv;
- uint32_t status;
+ uint32_t len;
- mutex_enter(&sc->sc_intr_lock);
- status = MMCI_READ(sc, MMCI_STATUS_REG);
-#ifdef PLMMC_DEBUG
- printf("%s: MMCI_STATUS_REG = %#x\n", __func__, status);
-#endif
- if (!status) {
- mutex_exit(&sc->sc_intr_lock);
- return 0;
+ if (cmd == NULL) {
+ device_printf(sc->sc_dev, "TX/RX interrupt with no active transfer\n");
+ return EINVAL;
}
- sc->sc_intr_status |= status;
- cv_broadcast(&sc->sc_intr_cv);
+ if (cmd->c_buf == NULL) {
+ return EINVAL;
+ }
+
+ const uint32_t fifo_cnt =
+ __SHIFTOUT(MMCI_READ(sc, MMCI_FIFO_CNT_REG), MMCI_FIFO_CNT) * 4;
+ if (fifo_cnt > sc->sc_fifo_resid) {
+ device_printf(sc->sc_dev, "FIFO counter is out of sync with active transfer\n");
+ return EIO;
+ }
- mutex_exit(&sc->sc_intr_lock);
+ if (cmd->c_flags & SCF_CMD_READ)
+ len = sc->sc_fifo_resid - fifo_cnt;
+ else
+ len = min(sc->sc_fifo_resid, PL181_FIFO_DEPTH);
- return 1;
+ if (len == 0)
+ return 0;
+
+ if (cmd->c_flags & SCF_CMD_READ)
+ MMCI_READ_MULTI(sc, MMCI_FIFO_REG, (uint32_t *)cmd->c_buf, len / 4);
+ else
+ MMCI_WRITE_MULTI(sc, MMCI_FIFO_REG, (uint32_t *)cmd->c_buf, len / 4);
+
+ sc->sc_fifo_resid -= len;
+ cmd->c_resid -= len;
+ cmd->c_buf += len;
+
+ return 0;
}
-static int
-plmmc_wait_status(struct plmmc_softc *sc, uint32_t mask, int timeout)
+int
+plmmc_intr(void *priv)
{
- int retry, error;
+ struct plmmc_softc *sc = priv;
+ uint32_t status, mask;
+ int retry = 100000;
- KASSERT(mutex_owned(&sc->sc_intr_lock));
+ mutex_enter(&sc->sc_lock);
- if (sc->sc_intr_status & mask)
- return 0;
+ while (--retry > 0) {
+ status = MMCI_READ(sc, MMCI_STATUS_REG);
+#ifdef PLMMC_DEBUG
+ printf("%s: MMCI_STATUS_REG = %#x\n", __func__, status);
+#endif
+ if ((status & sc->sc_status_mask) == 0)
+ break;
+ MMCI_WRITE(sc, MMCI_CLEAR_REG, status);
+ sc->sc_intr_status |= status;
- retry = timeout / hz;
- if (sc->sc_ih == NULL)
- retry *= 1000;
-
- while (retry > 0) {
- if (sc->sc_ih == NULL) {
- sc->sc_intr_status |= MMCI_READ(sc, MMCI_STATUS_REG);
- if (sc->sc_intr_status & mask)
- return 0;
- delay(10000);
- } else {
- error = cv_timedwait(&sc->sc_intr_cv,
- &sc->sc_intr_lock, hz);
- if (error && error != EWOULDBLOCK) {
- device_printf(sc->sc_dev,
- "cv_timedwait returned %d\n", error);
- return error;
+ if (status & MMCI_INT_CMD_TIMEOUT)
+ break;
+
+ if (status & (MMCI_INT_DATA_TIMEOUT|MMCI_INT_DATA_CRC_FAIL)) {
+ device_printf(sc->sc_dev,
+ "data xfer error, status %08x\n", status);
+ break;
+ }
+
+ if (status & (MMCI_INT_TX_FIFO_EMPTY|MMCI_INT_TX_FIFO_HALF_EMPTY|
+ MMCI_INT_RX_FIFO_FULL|MMCI_INT_RX_FIFO_HALF_FULL|
+ MMCI_INT_DATA_END|MMCI_INT_DATA_BLOCK_END)) {
+
+ /* Data transfer in progress */
+ if (plmmc_intr_xfer(sc, sc->sc_cmd) == 0 &&
+ sc->sc_fifo_resid == 0) {
+ /* Disable data IRQs */
+ mask = MMCI_READ(sc, MMCI_MASK0_REG);
+ mask &= ~PLMMC_INT_DATA_MASK;
+ MMCI_WRITE(sc, MMCI_MASK0_REG, mask);
+ /* Ignore data status bits after transfer */
+ sc->sc_status_mask &= ~PLMMC_INT_DATA_MASK;
}
- if (sc->sc_intr_status & mask)
- return 0;
}
- --retry;
+
+ if (status & MMCI_INT_CMD_RESP_END)
+ cv_broadcast(&sc->sc_intr_cv);
+ }
+ if (retry == 0) {
+ device_printf(sc->sc_dev, "intr handler stuck, fifo resid %d, status %08x\n",
+ sc->sc_fifo_resid, MMCI_READ(sc, MMCI_STATUS_REG));
}
- device_printf(sc->sc_dev, "%s timeout, MMCI_STATUS_REG = %#x\n",
- __func__, MMCI_READ(sc, MMCI_STATUS_REG));
+ cv_broadcast(&sc->sc_intr_cv);
+ mutex_exit(&sc->sc_lock);
- return ETIMEDOUT;
+ return 1;
}
static int
-plmmc_pio_wait(struct plmmc_softc *sc, struct sdmmc_command *cmd)
+plmmc_wait_cmd(struct plmmc_softc *sc)
{
- uint32_t bit = (cmd->c_flags & SCF_CMD_READ) ?
- MMCI_INT_RX_DATA_AVAIL : MMCI_INT_TX_FIFO_EMPTY;
+ int error = 0;
+
+ KASSERT(mutex_owned(&sc->sc_lock));
- MMCI_WRITE(sc, MMCI_CLEAR_REG, bit);
- const int error = plmmc_wait_status(sc,
- bit | MMCI_INT_DATA_END | MMCI_INT_DATA_BLOCK_END, hz*2);
- sc->sc_intr_status &= ~bit;
+ while (error == 0) {
+ if (sc->sc_intr_status & MMCI_INT_CMD_TIMEOUT) {
+ error = ETIMEDOUT;
+ break;
+ } else if (sc->sc_intr_status & MMCI_INT_CMD_RESP_END) {
+ break;
+ }
+
+ error = cv_timedwait(&sc->sc_intr_cv, &sc->sc_lock, hz * 2);
+ if (error != 0)
+ break;
+ }
return error;
}
@@ -212,22 +274,21 @@ static int
plmmc_pio_transfer(struct plmmc_softc *sc, struct sdmmc_command *cmd,
int xferlen)
{
- uint32_t *datap = (uint32_t *)cmd->c_buf;
- int i;
+ int error = 0;
- for (i = 0; i < xferlen / 4; i++) {
- if (plmmc_pio_wait(sc, cmd))
- return ETIMEDOUT;
- if (cmd->c_flags & SCF_CMD_READ) {
- datap[i] = MMCI_READ(sc, MMCI_FIFO_REG);
- } else {
- MMCI_WRITE(sc, MMCI_FIFO_REG, datap[i]);
- }
- cmd->c_resid -= 4;
- cmd->c_buf += 4;
+ while (sc->sc_fifo_resid > 0 && error == 0) {
+ error = cv_timedwait(&sc->sc_intr_cv,
+ &sc->sc_lock, hz * 5);
+ if (error != 0)
+ break;
+
+ if (sc->sc_intr_status & MMCI_INT_DATA_TIMEOUT)
+ error = ETIMEDOUT;
+ else if (sc->sc_intr_status & MMCI_INT_DATA_CRC_FAIL)
+ error = EIO;
}
- return 0;
+ return error;
}
static int
@@ -321,10 +382,15 @@ plmmc_do_command(sdmmc_chipset_handle_t
struct plmmc_softc *sc = sch;
uint32_t cmdval = MMCI_COMMAND_ENABLE;
- KASSERT(mutex_owned(&sc->sc_intr_lock));
+ KASSERT(mutex_owned(&sc->sc_lock));
const int xferlen = min(cmd->c_resid, PLMMC_MAXXFER);
+ sc->sc_cmd = cmd;
+ sc->sc_fifo_resid = xferlen;
+ sc->sc_status_mask = ~0U;
+ sc->sc_intr_status = 0;
+
#ifdef PLMMC_DEBUG
device_printf(sc->sc_dev,
"opcode %d flags %#x datalen %d resid %d xferlen %d\n",
@@ -334,13 +400,7 @@ plmmc_do_command(sdmmc_chipset_handle_t
MMCI_WRITE(sc, MMCI_COMMAND_REG, 0);
MMCI_WRITE(sc, MMCI_MASK0_REG, 0);
MMCI_WRITE(sc, MMCI_CLEAR_REG, 0xffffffff);
- MMCI_WRITE(sc, MMCI_MASK0_REG,
- MMCI_INT_CMD_TIMEOUT | MMCI_INT_DATA_TIMEOUT |
- MMCI_INT_RX_DATA_AVAIL | MMCI_INT_TX_FIFO_EMPTY |
- MMCI_INT_DATA_END | MMCI_INT_DATA_BLOCK_END |
- MMCI_INT_CMD_RESP_END | MMCI_INT_CMD_SENT);
-
- sc->sc_intr_status = 0;
+ MMCI_WRITE(sc, MMCI_MASK0_REG, PLMMC_INT_DATA_MASK | PLMMC_INT_CMD_MASK);
if (cmd->c_flags & SCF_RSP_PRESENT)
cmdval |= MMCI_COMMAND_RESPONSE;
@@ -378,6 +438,10 @@ plmmc_do_command(sdmmc_chipset_handle_t
if (xferlen > 0) {
cmd->c_error = plmmc_pio_transfer(sc, cmd, xferlen);
if (cmd->c_error) {
+#ifdef PLMMC_DEBUG
+ device_printf(sc->sc_dev,
+ "MMCI_STATUS_REG = %08x\n", MMCI_READ(sc, MMCI_STATUS_REG));
+#endif
device_printf(sc->sc_dev,
"error (%d) waiting for xfer\n", cmd->c_error);
goto done;
@@ -385,12 +449,7 @@ plmmc_do_command(sdmmc_chipset_handle_t
}
if ((cmd->c_flags & SCF_RSP_PRESENT) && cmd->c_resid == 0) {
- cmd->c_error = plmmc_wait_status(sc,
- MMCI_INT_CMD_RESP_END|MMCI_INT_CMD_TIMEOUT, hz * 2);
- if (cmd->c_error == 0 &&
- (sc->sc_intr_status & MMCI_INT_CMD_TIMEOUT)) {
- cmd->c_error = ETIMEDOUT;
- }
+ cmd->c_error = plmmc_wait_cmd(sc);
if (cmd->c_error) {
#ifdef PLMMC_DEBUG
device_printf(sc->sc_dev,
@@ -419,14 +478,15 @@ plmmc_do_command(sdmmc_chipset_handle_t
}
done:
+ sc->sc_cmd = NULL;
+
MMCI_WRITE(sc, MMCI_COMMAND_REG, 0);
MMCI_WRITE(sc, MMCI_MASK0_REG, 0);
MMCI_WRITE(sc, MMCI_CLEAR_REG, 0xffffffff);
MMCI_WRITE(sc, MMCI_DATA_CNT_REG, 0);
#ifdef PLMMC_DEBUG
- device_printf(sc->sc_dev, "MMCI_STATUS_REG = %#x\n",
- MMCI_READ(sc, MMCI_STATUS_REG));
+ device_printf(sc->sc_dev, "status = %#x\n", sc->sc_intr_status);
#endif
}
@@ -440,7 +500,7 @@ plmmc_exec_command(sdmmc_chipset_handle_
cmd->c_opcode, cmd->c_flags, cmd->c_data, cmd->c_datalen);
#endif
- mutex_enter(&sc->sc_intr_lock);
+ mutex_enter(&sc->sc_lock);
cmd->c_resid = cmd->c_datalen;
cmd->c_buf = cmd->c_data;
do {
@@ -460,7 +520,7 @@ plmmc_exec_command(sdmmc_chipset_handle_
}
} while (cmd->c_resid > 0 && cmd->c_error == 0);
cmd->c_flags |= SCF_ITSDONE;
- mutex_exit(&sc->sc_intr_lock);
+ mutex_exit(&sc->sc_lock);
}
static void
Index: src/sys/dev/ic/pl181var.h
diff -u src/sys/dev/ic/pl181var.h:1.2 src/sys/dev/ic/pl181var.h:1.3
--- src/sys/dev/ic/pl181var.h:1.2 Fri Jun 2 11:01:15 2017
+++ src/sys/dev/ic/pl181var.h Mon Feb 19 19:00:42 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: pl181var.h,v 1.2 2017/06/02 11:01:15 jmcneill Exp $ */
+/* $NetBSD: pl181var.h,v 1.3 2018/02/19 19:00:42 jmcneill Exp $ */
/*-
* Copyright (c) 2015 Jared D. McNeill <jmcne...@invisible.ca>
@@ -33,15 +33,17 @@ struct plmmc_softc {
device_t sc_dev;
bus_space_tag_t sc_bst;
bus_space_handle_t sc_bsh;
- void *sc_ih;
unsigned int sc_clock_freq;
unsigned int sc_max_freq;
device_t sc_sdmmc_dev;
- kmutex_t sc_intr_lock;
+ kmutex_t sc_lock;
kcondvar_t sc_intr_cv;
-
uint32_t sc_intr_status;
+
+ struct sdmmc_command *sc_cmd;
+ int sc_fifo_resid;
+ uint32_t sc_status_mask;
};
void plmmc_init(struct plmmc_softc *);
