--- drivers/usb/usb-ohci.h-0323 Fri Mar 23 10:13:17 2001 +++ drivers/usb/usb-ohci.h Sat Mar 24 13:04:32 2001 @@ -78,7 +78,7 @@ #define TD_DP_IN 0x00100000 #define TD_DP_OUT 0x00080000 -#define TD_ISO 0x00010000 +#define TD_ISO 0x00010000 #define TD_DEL 0x00020000 /* CC Codes */ @@ -113,11 +113,11 @@ urb_t * urb; dma_addr_t td_dma; - __u32 unused2[3]; + dma_addr_t data_dma; + __u32 unused2[2]; } __attribute((aligned(16))); typedef struct td td_t; - #define OHCI_ED_SKIP (1 << 14) /* @@ -344,6 +344,27 @@ } urb_priv_t; #define URB_DEL 1 + +/* Hash struct used for TD/ED hashing */ +struct hash_t { + void *virt; + dma_addr_t dma; + struct hash_t *next; // chaining for collision cases +}; + +/* List of TD/ED hash entries */ +struct hash_list_t { + struct hash_t *head; + struct hash_t *tail; +}; + +#define TD_HASH_SIZE 64 /* power'o'two */ +#define ED_HASH_SIZE 64 /* power'o'two */ + +#define TD_HASH_FUNC(td_dma) ((td_dma ^ (td_dma >> 5)) % TD_HASH_SIZE) +#define ED_HASH_FUNC(ed_dma) ((ed_dma ^ (ed_dma >> 5)) % ED_HASH_SIZE) + + /* * This is the full ohci controller description * @@ -383,6 +404,9 @@ u8 pci_latency; struct pci_pool *td_cache; struct pci_pool *dev_cache; + struct hash_list_t td_hash[TD_HASH_SIZE]; + struct hash_list_t ed_hash[ED_HASH_SIZE]; + } ohci_t; #define NUM_EDS 32 /* num of preallocated endpoint descriptors */ @@ -404,7 +428,7 @@ static ed_t * ep_add_ed(struct usb_device * usb_dev, unsigned int pipe, int interval, int load, int mem_flags); static void ep_rm_ed(struct usb_device * usb_dev, ed_t * ed); /* td */ -static void td_fill(unsigned int info, void * data, int len, urb_t * urb, int index); +static void td_fill(ohci_t * ohci, unsigned int info, dma_addr_t data, int len, urb_t * urb, int index); static void td_submit_urb(urb_t * urb); /* root hub */ static int rh_submit_urb(urb_t * urb); @@ -426,6 +450,110 @@ /* to support non-PCI OHCIs, you need custom bus/mem/... glue */ #endif + +/* Recover a TD/ED using its collision chain */ +static inline void * +dma_to_ed_td (struct hash_list_t * entry, dma_addr_t dma) +{ + struct hash_t * scan = entry->head; + while (scan && scan->dma != dma) + scan = scan->next; + if (!scan) + BUG(); + return scan->virt; +} + +static inline struct ed * +dma_to_ed (struct ohci * hc, dma_addr_t ed_dma) +{ + return (struct ed *) dma_to_ed_td(&(hc->ed_hash[ED_HASH_FUNC(ed_dma)]), + ed_dma); +} + +static inline struct td * +dma_to_td (struct ohci * hc, dma_addr_t td_dma) +{ + return (struct td *) dma_to_ed_td(&(hc->td_hash[TD_HASH_FUNC(td_dma)]), + td_dma); +} + +/* Add a hash entry for a TD/ED; return true on success */ +static inline int +hash_add_ed_td(struct hash_list_t * entry, void * virt, dma_addr_t dma) +{ + struct hash_t * scan; + + scan = (struct hash_t *)kmalloc(sizeof(struct hash_t), ALLOC_FLAGS); + if (!scan) + return 0; + + if (!entry->tail) { + entry->head = entry->tail = scan; + } else { + entry->tail->next = scan; + entry->tail = scan; + } + + scan->virt = virt; + scan->dma = dma; + scan->next = NULL; + return 1; +} + +static inline int +hash_add_ed (struct ohci * hc, struct ed * ed) +{ + return hash_add_ed_td (&(hc->ed_hash[ED_HASH_FUNC(ed->dma)]), + ed, ed->dma); +} + +static inline int +hash_add_td (struct ohci * hc, struct td * td) +{ + return hash_add_ed_td (&(hc->td_hash[TD_HASH_FUNC(td->td_dma)]), + td, td->td_dma); +} + + +static inline void +hash_free_ed_td (struct hash_list_t * entry, void * virt) +{ + struct hash_t *scan, *prev; + scan = prev = entry->head; + + // Find and unlink hash entry + while (scan && scan->virt != virt) { + prev = scan; + scan = scan->next; + } + if (scan) { + if (scan == entry->head) { + if (entry->head == entry->tail) + entry->head = entry->tail = NULL; + else + entry->head = scan->next; + } else if (scan == entry->tail) { + entry->tail = prev; + prev->next = NULL; + } else + prev->next = scan->next; + kfree(scan); + } +} + +static inline void +hash_free_ed (struct ohci * hc, struct ed * ed) +{ + hash_free_ed_td (&(hc->ed_hash[ED_HASH_FUNC(ed->dma)]), ed); +} + +static inline void +hash_free_td (struct ohci * hc, struct td * td) +{ + hash_free_ed_td (&(hc->td_hash[TD_HASH_FUNC(td->td_dma)]), td); +} + + static int ohci_mem_init (struct ohci *ohci) { ohci->td_cache = pci_pool_create ("ohci_td", ohci->ohci_dev, @@ -465,14 +593,22 @@ struct td *td; td = pci_pool_alloc (hc->td_cache, mem_flags, &dma); - if (td) + if (td) { td->td_dma = dma; + + /* hash it for later reverse mapping */ + if (!hash_add_td (hc, td)) { + pci_pool_free (hc->td_cache, td, dma); + return NULL; + } + } return td; } static inline void td_free (struct ohci *hc, struct td *td) { + hash_free_td (hc, td); pci_pool_free (hc->td_cache, td, td->td_dma); } @@ -492,6 +628,7 @@ offset = ((char *)&dev->ed) - ((char *)dev); for (i = 0; i < NUM_EDS; i++, offset += sizeof dev->ed [0]) dev->ed [i].dma = dma + offset; + /* add to hashtable if used */ } return dev; } --- drivers/usb/usb-ohci.c-0323 Wed Mar 21 17:33:06 2001 +++ drivers/usb/usb-ohci.c Sat Mar 24 13:27:13 2001 @@ -12,8 +12,11 @@ * * History: * + * 2001/03/24 pci_map_single (db) + * 2001/03/23 td/ed hashing to remove bus_to_virt (Steve Longerbeam) * 2001/03/21 td and dev/ed allocation uses new pci_pool API (db) * 2001/03/07 hcca allocation uses pci_alloc_consistent (Steve Longerbeam) + * * 2000/09/26 fixed races in removing the private portion of the urb * 2000/09/07 disable bulk and control lists when unlinking the last * endpoint descriptor in order to avoid unrecoverable errors on @@ -99,10 +102,29 @@ static void urb_free_priv (struct ohci *hc, urb_priv_t * urb_priv) { - int i; - - for (i = 0; i < urb_priv->length; i++) { - if (urb_priv->td [i]) { + int i; + int last = urb_priv->length - 1; + struct td *td; + + for (i = 0; i <= last; i++) { + td = urb_priv->td [i]; + if (td) { + int len; + int dir; + + if ((td->hwINFO & cpu_to_le32 (TD_DP)) == TD_DP_SETUP) { + len = 8; + dir = PCI_DMA_TODEVICE; + } else if (i == last) { + len = td->urb->transfer_buffer_length, + dir = usb_pipeout (td->urb->pipe) + ? PCI_DMA_TODEVICE + : PCI_DMA_FROMDEVICE; + } else + len = dir = 0; + if (len && td->data_dma) + pci_unmap_single (hc->ohci_dev, + td->data_dma, len, dir); td_free (hc, urb_priv->td [i]); } } @@ -213,7 +235,7 @@ continue; printk (KERN_DEBUG __FILE__ ": %s branch int %2d(%2x):", str, i, i); while (*ed_p != 0 && j--) { - ed_t *ed = (ed_t *) bus_to_virt(le32_to_cpup(ed_p)); + ed_t *ed = dma_to_ed (ohci, le32_to_cpup(ed_p)); printk (" ed: %4x;", ed->hwINFO); ed_p = &ed->hwNextED; } @@ -384,7 +406,7 @@ /* return a request to the completion handler */ -static int sohci_return_urb (urb_t * urb) +static int sohci_return_urb (struct ohci *hc, urb_t * urb) { urb_priv_t * urb_priv = urb->hcpriv; urb_t * urbt; @@ -406,7 +428,15 @@ switch (usb_pipetype (urb->pipe)) { case PIPE_INTERRUPT: - urb->complete (urb); /* call complete and requeue URB */ + pci_unmap_single (hc->ohci_dev, + urb_priv->td [0]->data_dma, + urb->transfer_buffer_length, + usb_pipeout (urb->pipe) + ? PCI_DMA_TODEVICE + : PCI_DMA_FROMDEVICE); + urb->complete (urb); + + /* implicitly requeued */ urb->actual_length = 0; urb->status = USB_ST_URB_PENDING; if (urb_priv->state != URB_DEL) @@ -416,8 +446,13 @@ case PIPE_ISOCHRONOUS: for (urbt = urb->next; urbt && (urbt != urb); urbt = urbt->next); if (urbt) { /* send the reply and requeue URB */ + pci_unmap_single (hc->ohci_dev, + urb_priv->td [0]->data_dma, + urb->transfer_buffer_length, + usb_pipeout (urb->pipe) + ? PCI_DMA_TODEVICE + : PCI_DMA_FROMDEVICE); urb->complete (urb); - spin_lock_irqsave (&usb_ed_lock, flags); urb->actual_length = 0; urb->status = USB_ST_URB_PENDING; @@ -910,9 +945,9 @@ for (i = 0; i < ep_rev (6, interval); i += inter) { inter = 1; for (ed_p = &(ohci->hcca->int_table[ep_rev (5, i) + int_branch]); - (*ed_p != 0) && (((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->int_interval >= interval); - ed_p = &(((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->hwNextED)) - inter = ep_rev (6, ((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->int_interval); + (*ed_p != 0) && ((dma_to_ed (ohci, le32_to_cpup (ed_p)))->int_interval >= interval); + ed_p = &((dma_to_ed (ohci, le32_to_cpup (ed_p)))->hwNextED)) + inter = ep_rev (6, (dma_to_ed (ohci, le32_to_cpup (ed_p)))->int_interval); ed->hwNextED = *ed_p; *ed_p = cpu_to_le32 (ed->dma); } @@ -932,8 +967,8 @@ inter = 1; for (ed_p = &(ohci->hcca->int_table[ep_rev (5, i)]); *ed_p != 0; - ed_p = &(((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->hwNextED)) - inter = ep_rev (6, ((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->int_interval); + ed_p = &((dma_to_ed (ohci, le32_to_cpup (ed_p)))->hwNextED)) + inter = ep_rev (6, (dma_to_ed (ohci, le32_to_cpup (ed_p)))->int_interval); *ed_p = cpu_to_le32 (ed->dma); } ed->ed_prev = NULL; @@ -978,7 +1013,7 @@ if (ohci->ed_controltail == ed) { ohci->ed_controltail = ed->ed_prev; } else { - ((ed_t *) bus_to_virt (le32_to_cpup (&ed->hwNextED)))->ed_prev = ed->ed_prev; + (dma_to_ed (ohci, le32_to_cpup (&ed->hwNextED)))->ed_prev = ed->ed_prev; } break; @@ -995,7 +1030,7 @@ if (ohci->ed_bulktail == ed) { ohci->ed_bulktail = ed->ed_prev; } else { - ((ed_t *) bus_to_virt (le32_to_cpup (&ed->hwNextED)))->ed_prev = ed->ed_prev; + (dma_to_ed (ohci, le32_to_cpup (&ed->hwNextED)))->ed_prev = ed->ed_prev; } break; @@ -1006,9 +1041,9 @@ for (i = 0; i < ep_rev (6, interval); i += inter) { for (ed_p = &(ohci->hcca->int_table[ep_rev (5, i) + int_branch]), inter = 1; (*ed_p != 0) && (*ed_p != ed->hwNextED); - ed_p = &(((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->hwNextED), - inter = ep_rev (6, ((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->int_interval)) { - if(((ed_t *) bus_to_virt (le32_to_cpup (ed_p))) == ed) { + ed_p = &((dma_to_ed (ohci, le32_to_cpup (ed_p)))->hwNextED), + inter = ep_rev (6, (dma_to_ed (ohci, le32_to_cpup (ed_p)))->int_interval)) { + if((dma_to_ed (ohci, le32_to_cpup (ed_p))) == ed) { *ed_p = ed->hwNextED; break; } @@ -1025,7 +1060,7 @@ if (ohci->ed_isotail == ed) ohci->ed_isotail = ed->ed_prev; if (ed->hwNextED != 0) - ((ed_t *) bus_to_virt (le32_to_cpup (&ed->hwNextED)))->ed_prev = ed->ed_prev; + (dma_to_ed (ohci, le32_to_cpup (&ed->hwNextED)))->ed_prev = ed->ed_prev; if (ed->ed_prev != NULL) { ed->ed_prev->hwNextED = ed->hwNextED; @@ -1033,9 +1068,9 @@ for (i = 0; i < 32; i++) { for (ed_p = &(ohci->hcca->int_table[ep_rev (5, i)]); *ed_p != 0; - ed_p = &(((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->hwNextED)) { - // inter = ep_rev (6, ((ed_t *) bus_to_virt (le32_to_cpup (ed_p)))->int_interval); - if(((ed_t *) bus_to_virt (le32_to_cpup (ed_p))) == ed) { + ed_p = &((dma_to_ed (ohci, le32_to_cpup (ed_p)))->hwNextED)) { + // inter = ep_rev (6, (dma_to_ed (ohci, le32_to_cpup (ed_p)))->int_interval); + if((dma_to_ed (ohci, le32_to_cpup (ed_p))) == ed) { *ed_p = ed->hwNextED; break; } @@ -1090,8 +1125,11 @@ ed->hwINFO = cpu_to_le32 (OHCI_ED_SKIP); /* skip ed */ /* dummy td; end of td list for ed */ td = td_alloc (ohci, mem_flags); - if (!td) { + /* hash the ed for later reverse mapping */ + if (!td || !hash_add_ed (ohci, (ed_t *)ed)) { /* out of memory */ + if (td) + td_free(ohci, td); spin_unlock_irqrestore (&usb_ed_lock, flags); return NULL; } @@ -1166,11 +1204,14 @@ /* enqueue next TD for this URB (OHCI spec 5.2.8.2) */ -static void td_fill (unsigned int info, void * data, int len, urb_t * urb, int index) +static void +td_fill (ohci_t * ohci, unsigned int info, + dma_addr_t data, int len, + urb_t * urb, int index) { volatile td_t * td, * td_pt; urb_priv_t * urb_priv = urb->hcpriv; - + if (index >= urb_priv->length) { err("internal OHCI error: TD index > length"); return; @@ -1181,29 +1222,30 @@ td_pt->hwNextTD = 0; /* fill the old dummy TD */ - td = urb_priv->td [index] = (td_t *) - bus_to_virt (le32_to_cpup (&urb_priv->ed->hwTailP) & 0xfffffff0); + td = urb_priv->td [index] = dma_to_td (ohci, + le32_to_cpup (&urb_priv->ed->hwTailP) & ~0xf); + td->ed = urb_priv->ed; td->next_dl_td = NULL; td->index = index; td->urb = urb; + td->data_dma = data; + if (!len) + data = 0; + td->hwINFO = cpu_to_le32 (info); if ((td->ed->type) == PIPE_ISOCHRONOUS) { - td->hwCBP = cpu_to_le32 (((!data || !len) - ? 0 - : virt_to_bus (data)) & 0xFFFFF000); + td->hwCBP = cpu_to_le32 (data & 0xFFFFF000); td->ed->last_iso = info & 0xffff; } else { - td->hwCBP = cpu_to_le32 (((!data || !len) - ? 0 - : virt_to_bus (data))); + td->hwCBP = cpu_to_le32 (data); } - td->hwBE = cpu_to_le32 ((!data || !len ) - ? 0 - : virt_to_bus (data + len - 1)); + if (data) + td->hwBE = cpu_to_le32 (data + len - 1); + else + td->hwBE = 0; td->hwNextTD = cpu_to_le32 (td_pt->td_dma); - - td->hwPSW [0] = cpu_to_le16 ((virt_to_bus (data) & 0x0FFF) | 0xE000); + td->hwPSW [0] = cpu_to_le16 ((data & 0x0FFF) | 0xE000); /* append to queue */ td->ed->hwTailP = td->hwNextTD; @@ -1217,8 +1259,7 @@ { urb_priv_t * urb_priv = urb->hcpriv; ohci_t * ohci = (ohci_t *) urb->dev->bus->hcpriv; - void * ctrl = urb->setup_packet; - void * data = urb->transfer_buffer; + dma_addr_t data; int data_len = urb->transfer_buffer_length; int cnt = 0; __u32 info = 0; @@ -1233,18 +1274,28 @@ } urb_priv->td_cnt = 0; + + if (data_len) { + data = pci_map_single (ohci->ohci_dev, + urb->transfer_buffer, data_len, + usb_pipeout (urb->pipe) + ? PCI_DMA_TODEVICE + : PCI_DMA_FROMDEVICE + ); + } else + data = 0; switch (usb_pipetype (urb->pipe)) { case PIPE_BULK: info = usb_pipeout (urb->pipe)? TD_CC | TD_DP_OUT : TD_CC | TD_DP_IN ; while(data_len > 4096) { - td_fill (info | (cnt? TD_T_TOGGLE:toggle), data, 4096, urb, cnt); + td_fill (ohci, info | (cnt? TD_T_TOGGLE:toggle), data, 4096, urb, cnt); data += 4096; data_len -= 4096; cnt++; } info = usb_pipeout (urb->pipe)? TD_CC | TD_DP_OUT : TD_CC | TD_R | TD_DP_IN ; - td_fill (info | (cnt? TD_T_TOGGLE:toggle), data, data_len, urb, cnt); + td_fill (ohci, info | (cnt? TD_T_TOGGLE:toggle), data, data_len, urb, cnt); cnt++; writel (OHCI_BLF, &ohci->regs->cmdstatus); /* start bulk list */ break; @@ -1252,27 +1303,32 @@ case PIPE_INTERRUPT: info = usb_pipeout (urb->pipe)? TD_CC | TD_DP_OUT | toggle: TD_CC | TD_R | TD_DP_IN | toggle; - td_fill (info, data, data_len, urb, cnt++); + td_fill (ohci, info, data, data_len, urb, cnt++); break; case PIPE_CONTROL: info = TD_CC | TD_DP_SETUP | TD_T_DATA0; - td_fill (info, ctrl, 8, urb, cnt++); + td_fill (ohci, info, + pci_map_single (ohci->ohci_dev, + urb->setup_packet, 8, + PCI_DMA_TODEVICE), + 8, urb, cnt++); if (data_len > 0) { info = usb_pipeout (urb->pipe)? TD_CC | TD_R | TD_DP_OUT | TD_T_DATA1 : TD_CC | TD_R | TD_DP_IN | TD_T_DATA1; - td_fill (info, data, data_len, urb, cnt++); + /* NOTE: mishandles transfers >8K, some >4K */ + td_fill (ohci, info, data, data_len, urb, cnt++); } info = usb_pipeout (urb->pipe)? TD_CC | TD_DP_IN | TD_T_DATA1: TD_CC | TD_DP_OUT | TD_T_DATA1; - td_fill (info, NULL, 0, urb, cnt++); + td_fill (ohci, info, data, 0, urb, cnt++); writel (OHCI_CLF, &ohci->regs->cmdstatus); /* start Control list */ break; case PIPE_ISOCHRONOUS: for (cnt = 0; cnt < urb->number_of_packets; cnt++) { - td_fill (TD_CC|TD_ISO | ((urb->start_frame + cnt) & 0xffff), - (__u8 *) data + urb->iso_frame_desc[cnt].offset, + td_fill (ohci, TD_CC|TD_ISO | ((urb->start_frame + cnt) & 0xffff), + data + urb->iso_frame_desc[cnt].offset, urb->iso_frame_desc[cnt].length, urb, cnt); } break; @@ -1323,9 +1379,9 @@ ((td->index == 0) || (td->index == urb_priv->length - 1)))) { if (tdBE != 0) { if (td->hwCBP == 0) - urb->actual_length = bus_to_virt (tdBE) - urb->transfer_buffer + 1; + urb->actual_length += tdBE - td->data_dma + 1; else - urb->actual_length = bus_to_virt (tdCBP) - urb->transfer_buffer; + urb->actual_length += tdCBP - td->data_dma; } } } @@ -1371,7 +1427,7 @@ ohci->hcca->done_head = 0; while (td_list_hc) { - td_list = (td_t *) bus_to_virt (td_list_hc); + td_list = dma_to_td (ohci, td_list_hc); if (TD_CC_GET (le32_to_cpup (&td_list->hwINFO))) { urb_priv = (urb_priv_t *) td_list->urb->hcpriv; @@ -1416,8 +1472,8 @@ for (ed = ohci->ed_rm_list[frame]; ed != NULL; ed = ed->ed_rm_list) { - tdTailP = bus_to_virt (le32_to_cpup (&ed->hwTailP) & 0xfffffff0); - tdHeadP = bus_to_virt (le32_to_cpup (&ed->hwHeadP) & 0xfffffff0); + tdTailP = dma_to_td (ohci, le32_to_cpup (&ed->hwTailP) & 0xfffffff0); + tdHeadP = dma_to_td (ohci, le32_to_cpup (&ed->hwHeadP) & 0xfffffff0); edINFO = le32_to_cpup (&ed->hwINFO); td_p = &ed->hwHeadP; @@ -1425,7 +1481,7 @@ urb_t * urb = td->urb; urb_priv_t * urb_priv = td->urb->hcpriv; - td_next = bus_to_virt (le32_to_cpup (&td->hwNextTD) & 0xfffffff0); + td_next = dma_to_td (ohci, le32_to_cpup (&td->hwNextTD) & 0xfffffff0); if ((urb_priv->state == URB_DEL) || (ed->state & ED_DEL)) { tdINFO = le32_to_cpup (&td->hwINFO); if (TD_CC_GET (tdINFO) < 0xE) @@ -1444,7 +1500,8 @@ struct ohci_device * dev = usb_to_ohci (ohci->dev[edINFO & 0x7F]); td_free (ohci, tdTailP); /* free dummy td */ ed->hwINFO = cpu_to_le32 (OHCI_ED_SKIP); - ed->state = ED_NEW; + ed->state = ED_NEW; + hash_free_ed(ohci, ed); /* if all eds are removed wake up sohci_free_dev */ if (!--dev->ed_cnt) { wait_queue_head_t *wait_head = dev->wait; @@ -1455,7 +1512,7 @@ } } else { ed->state &= ~ED_URB_DEL; - tdHeadP = bus_to_virt (le32_to_cpup (&ed->hwHeadP) & 0xfffffff0); + tdHeadP = dma_to_td (ohci, le32_to_cpup (&ed->hwHeadP) & 0xfffffff0); if (tdHeadP == tdTailP) { if (ed->state == ED_OPER) @@ -1463,6 +1520,7 @@ td_free (ohci, tdTailP); ed->hwINFO = cpu_to_le32 (OHCI_ED_SKIP); ed->state = ED_NEW; + hash_free_ed(ohci, ed); --(usb_to_ohci (ohci->dev[edINFO & 0x7F]))->ed_cnt; } else ed->hwINFO &= ~cpu_to_le32 (OHCI_ED_SKIP); @@ -1540,7 +1598,7 @@ if ((ed->state & (ED_OPER | ED_UNLINK)) && (urb_priv->state != URB_DEL)) { urb->status = cc_to_error[cc]; - sohci_return_urb (urb); + sohci_return_urb (ohci, urb); } else { spin_lock_irqsave (&usb_ed_lock, flags); dl_del_urb (urb);