On Mon, 11 Jul 2016, Reyk Floeter wrote:
> The intentional 4GB limit is for forwarding: what if you forward mbufs
> from a 64bit-capable interface to another one that doesn't support 64bit
> DMA? And even if you would only enable it if all interfaces are
> 64bit-capable, what if you plug in a 32bit USB/hotplug interface? We did
> not want to support bounce buffers in OpenBSD.
Yes, I have understood that. My mail was more about non-mbuf DMA: Does it
make sense to allow 64bit DMA in other cases while keeping the 4GB
limitation for mbufs?
Cheers,
Stefan
>
> Reyk
>
> > On 11.07.2016, at 15:37, Stefan Fritsch <s...@sfritsch.de> wrote:
> >
> > Hi,
> >
> > following the discussion about mbufs, I have some questions about 64bit
> > DMA in general.
> >
> > Openbsd on amd64 assumes that DMA is only possible to the lower 4GB. But
> > there are many devices (PCIe, virtio, ...) that can do DMA to the whole
> > memory. Is it feasible to have known good devices opt-in into 64bit DMA?
> >
> > I have done a patch that allows virtio to do 64bit DMA. This works insofar
> > as the queues used by the device will now be allocated above 4GB. But this
> > is only a small amount of memory. More interesting would be bufs and
> > mbufs.
> >
> >
> > For bufs, Bob added some code for copying bufs above/below 4GB. But this
> > code only has a single flag B_DMA to denote if DMA is possible into a buf
> > or not. Would it make sense to replace that by a mechanism that is device
> > specific, so that we can use devices efficiently that allow 64bit DMA?
> > Maybe a flag in the device vnode?
> >
> >
> > Does it make sense to commit something like the diff below (not tested
> > much), even if it saves at most a few MB below 4GB right now?
> >
> > Cheers,
> > Stefan
> >
> >
> > diff --git sys/arch/amd64/amd64/bus_dma.c sys/arch/amd64/amd64/bus_dma.c
> > index 8eaa2e7..1aba7c0 100644
> > --- sys/arch/amd64/amd64/bus_dma.c
> > +++ sys/arch/amd64/amd64/bus_dma.c
> > @@ -293,6 +293,7 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
> > bus_dma_segment_t *segs,
> > {
> > bus_addr_t paddr, baddr, bmask, lastaddr = 0;
> > bus_size_t plen, sgsize, mapsize;
> > + struct uvm_constraint_range *constraint = t->_cookie;
> > int first = 1;
> > int i, seg = 0;
> >
> > @@ -320,7 +321,7 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
> > bus_dma_segment_t *segs,
> > if (plen < sgsize)
> > sgsize = plen;
> >
> > - if (paddr > dma_constraint.ucr_high)
> > + if (paddr > constraint->ucr_high)
> > panic("Non dma-reachable buffer at paddr
> > %#lx(raw)",
> > paddr);
> >
> > @@ -405,15 +406,11 @@ _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size,
> > bus_size_t alignment,
> > bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
> > int flags)
> > {
> > + struct uvm_constraint_range *constraint = t->_cookie;
> >
> > - /*
> > - * XXX in the presence of decent (working) iommus and bouncebuffers
> > - * we can then fallback this allocation to a range of { 0, -1 }.
> > - * However for now we err on the side of caution and allocate dma
> > - * memory under the 4gig boundary.
> > - */
> > return (_bus_dmamem_alloc_range(t, size, alignment, boundary,
> > - segs, nsegs, rsegs, flags, (bus_addr_t)0, (bus_addr_t)0xffffffff));
> > + segs, nsegs, rsegs, flags, (bus_addr_t)constraint->ucr_low,
> > + (bus_addr_t)constraint->ucr_high));
> > }
> >
> > /*
> > @@ -567,6 +564,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t
> > map, void *buf,
> > bus_size_t sgsize;
> > bus_addr_t curaddr, lastaddr, baddr, bmask;
> > vaddr_t vaddr = (vaddr_t)buf;
> > + struct uvm_constraint_range *constraint = t->_cookie;
> > int seg;
> > pmap_t pmap;
> >
> > @@ -584,7 +582,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t
> > map, void *buf,
> > */
> > pmap_extract(pmap, vaddr, (paddr_t *)&curaddr);
> >
> > - if (curaddr > dma_constraint.ucr_high)
> > + if (curaddr > constraint->ucr_high)
> > panic("Non dma-reachable buffer at curaddr %#lx(raw)",
> > curaddr);
> >
> > diff --git sys/arch/amd64/amd64/machdep.c sys/arch/amd64/amd64/machdep.c
> > index de9f481..7640532 100644
> > --- sys/arch/amd64/amd64/machdep.c
> > +++ sys/arch/amd64/amd64/machdep.c
> > @@ -201,6 +201,12 @@ struct vm_map *phys_map = NULL;
> >
> > /* UVM constraint ranges. */
> > struct uvm_constraint_range isa_constraint = { 0x0, 0x00ffffffUL };
> > + /*
> > + * XXX in the presence of decent (working) iommus and bouncebuffers
> > + * we can then fallback this allocation to a range of { 0, -1 }.
> > + * However for now we err on the side of caution and allocate dma
> > + * memory under the 4gig boundary.
> > + */
> > struct uvm_constraint_range dma_constraint = { 0x0, 0xffffffffUL };
> > struct uvm_constraint_range *uvm_md_constraints[] = {
> > &isa_constraint,
> > diff --git sys/arch/amd64/include/pci_machdep.h
> > sys/arch/amd64/include/pci_machdep.h
> > index 27b833b..bf54f31 100644
> > --- sys/arch/amd64/include/pci_machdep.h
> > +++ sys/arch/amd64/include/pci_machdep.h
> > @@ -41,6 +41,7 @@
> > */
> >
> > extern struct bus_dma_tag pci_bus_dma_tag;
> > +extern struct bus_dma_tag virtio_pci_bus_dma_tag;
> >
> > /*
> > * Types provided to machine-independent PCI code
> > diff --git sys/arch/amd64/isa/isa_machdep.c sys/arch/amd64/isa/isa_machdep.c
> > index 74dc907..ec35edead 100644
> > --- sys/arch/amd64/isa/isa_machdep.c
> > +++ sys/arch/amd64/isa/isa_machdep.c
> > @@ -140,7 +140,7 @@ void _isa_dma_free_bouncebuf(bus_dma_tag_t,
> > bus_dmamap_t);
> > * buffers, if necessary.
> > */
> > struct bus_dma_tag isa_bus_dma_tag = {
> > - NULL, /* _cookie */
> > + &isa_constraint,
> > _isa_bus_dmamap_create,
> > _isa_bus_dmamap_destroy,
> > _isa_bus_dmamap_load,
> > diff --git sys/arch/amd64/pci/pci_machdep.c sys/arch/amd64/pci/pci_machdep.c
> > index 19983b0..5dca63a 100644
> > --- sys/arch/amd64/pci/pci_machdep.c
> > +++ sys/arch/amd64/pci/pci_machdep.c
> > @@ -130,7 +130,7 @@ do {
> > \
> > * of these functions.
> > */
> > struct bus_dma_tag pci_bus_dma_tag = {
> > - NULL, /* _may_bounce */
> > + &dma_constraint,
> > _bus_dmamap_create,
> > _bus_dmamap_destroy,
> > _bus_dmamap_load,
> > @@ -147,6 +147,25 @@ struct bus_dma_tag pci_bus_dma_tag = {
> > _bus_dmamem_mmap,
> > };
> >
> > +struct bus_dma_tag virtio_pci_bus_dma_tag = {
> > + &no_constraint,
> > + _bus_dmamap_create,
> > + _bus_dmamap_destroy,
> > + _bus_dmamap_load,
> > + _bus_dmamap_load_mbuf,
> > + _bus_dmamap_load_uio,
> > + _bus_dmamap_load_raw,
> > + _bus_dmamap_unload,
> > + _bus_dmamap_sync,
> > + _bus_dmamem_alloc,
> > + _bus_dmamem_alloc_range,
> > + _bus_dmamem_free,
> > + _bus_dmamem_map,
> > + _bus_dmamem_unmap,
> > + _bus_dmamem_mmap,
> > +};
> > +
> > +
> > void
> > pci_attach_hook(struct device *parent, struct device *self,
> > struct pcibus_attach_args *pba)
> > diff --git sys/arch/i386/pci/pci_machdep.h sys/arch/i386/pci/pci_machdep.h
> > index 2831470..6e6484b 100644
> > --- sys/arch/i386/pci/pci_machdep.h
> > +++ sys/arch/i386/pci/pci_machdep.h
> > @@ -56,6 +56,7 @@ union i386_pci_tag_u {
> > };
> >
> > extern struct bus_dma_tag pci_bus_dma_tag;
> > +#define virtio_pci_bus_dma_tag pci_bus_dma_tag
> >
> > /*
> > * Types provided to machine-independent PCI code
> > diff --git sys/dev/pci/virtio_pci.c sys/dev/pci/virtio_pci.c
> > index 489a449..6c08579 100644
> > --- sys/dev/pci/virtio_pci.c
> > +++ sys/dev/pci/virtio_pci.c
> > @@ -183,7 +183,7 @@ virtio_pci_attach(struct device *parent, struct device
> > *self, void *aux)
> >
> > vsc->sc_ops = &virtio_pci_ops;
> > sc->sc_pc = pc;
> > - vsc->sc_dmat = pa->pa_dmat;
> > + vsc->sc_dmat = &virtio_pci_bus_dma_tag;
> > sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
> >
> > if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
> >
>
>
