Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Sat, Aug 1, 2020 at 1:17 PM Nicolas Saenz Julienne
wrote:
>
> Hi Jim, here's some comments after testing your series against RPi4.
>
> On Fri, 2020-07-24 at 16:33 -0400, Jim Quinlan wrote:
> > The new field 'dma_range_map' in struct device is used to facilitate the
> > use of single or multiple offsets between mapping regions of cpu addrs and
> > dma addrs. It subsumes the role of "dev->dma_pfn_offset" which was only
> > capable of holding a single uniform offset and had no region bounds
> > checking.
> >
> > The function of_dma_get_range() has been modified so that it takes a single
> > argument -- the device node -- and returns a map, NULL, or an error code.
> > The map is an array that holds the information regarding the DMA regions.
> > Each range entry contains the address offset, the cpu_start address, the
> > dma_start address, and the size of the region.
> >
> > of_dma_configure() is the typical manner to set range offsets but there are
> > a number of ad hoc assignments to "dev->dma_pfn_offset" in the kernel
> > driver code. These cases now invoke the function
> > dma_attach_offset_range(dev, cpu_addr, dma_addr, size).
> >
> > Signed-off-by: Jim Quinlan
> > ---
>
> [...]
>
> > diff --git a/drivers/of/address.c b/drivers/of/address.c
> > index 8eea3f6e29a4..4b718d199efe 100644
> > --- a/drivers/of/address.c
> > +++ b/drivers/of/address.c
> > @@ -918,33 +918,33 @@ void __iomem *of_io_request_and_map(struct
> > device_node *np, int index,
> > }
> > EXPORT_SYMBOL(of_io_request_and_map);
> >
> > +#ifdef CONFIG_HAS_DMA
> > /**
> > - * of_dma_get_range - Get DMA range info
> > + * of_dma_get_range - Get DMA range info and put it into a map array
> > * @np: device node to get DMA range info
> > - * @dma_addr:pointer to store initial DMA address of DMA range
> > - * @paddr: pointer to store initial CPU address of DMA range
> > - * @size:pointer to store size of DMA range
> > + * @map: dma range structure to return
> > *
> > * Look in bottom up direction for the first "dma-ranges" property
> > - * and parse it.
> > - * dma-ranges format:
> > + * and parse it. Put the information into a DMA offset map array.
> > + *
> > + * dma-ranges format:
> > * DMA addr (dma_addr) : naddr cells
> > * CPU addr (phys_addr_t) : pna cells
> > * size: nsize cells
> > *
> > - * It returns -ENODEV if "dma-ranges" property was not found
> > - * for this device in DT.
> > + * It returns -ENODEV if "dma-ranges" property was not found for this
> > + * device in the DT.
> > */
> > -int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr,
> > u64 *size)
> > +int of_dma_get_range(struct device_node *np, const struct bus_dma_region
> > **map)
> > {
> > struct device_node *node = of_node_get(np);
> > const __be32 *ranges = NULL;
> > - int len;
> > - int ret = 0;
> > bool found_dma_ranges = false;
> > struct of_range_parser parser;
> > struct of_range range;
> > - u64 dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
> > + struct bus_dma_region *r;
> > + int len, num_ranges = 0;
> > + int ret;
> >
> > while (node) {
> > ranges = of_get_property(node, "dma-ranges", );
> > @@ -970,44 +970,35 @@ int of_dma_get_range(struct device_node *np, u64
> > *dma_addr, u64 *paddr, u64 *siz
> > }
> >
> > of_dma_range_parser_init(, node);
> > + for_each_of_range(, )
> > + num_ranges++;
> > +
> > + of_dma_range_parser_init(, node);
> > +
> > + ret = -ENOMEM;
> > + r = kcalloc(num_ranges + 1, sizeof(*r), GFP_KERNEL);
> > + if (!r)
> > + goto out;
> >
> > + /*
> > + * Record all info in the generic DMA ranges array for struct device.
> > + */
> > + *map = r;
> > for_each_of_range(, ) {
> > pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
> >range.bus_addr, range.cpu_addr, range.size);
> > -
> > - if (dma_offset && range.cpu_addr - range.bus_addr !=
> > dma_offset) {
> > - pr_warn("Can't handle multiple dma-ranges with
> > different offsets on node(%pOF)\n", node);
> > - /* Don't error out as we'd break some existing DTs */
> > - continue;
> > - }
> > - dma_offset = range.cpu_addr - range.bus_addr;
> > -
> > - /* Take lower and upper limits */
> > - if (range.bus_addr < dma_start)
> > - dma_start = range.bus_addr;
> > - if (range.bus_addr + range.size > dma_end)
> > - dma_end = range.bus_addr + range.size;
> > - }
> > -
> > - if (dma_start >= dma_end) {
> > - ret = -EINVAL;
> > - pr_debug("Invalid DMA ranges configuration on node(%pOF)\n",
> > - node);
> > - goto out;
> > + r->cpu_start = range.cpu_addr;
> >
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
Hi Jim, here's some comments after testing your series against RPi4.
On Fri, 2020-07-24 at 16:33 -0400, Jim Quinlan wrote:
> The new field 'dma_range_map' in struct device is used to facilitate the
> use of single or multiple offsets between mapping regions of cpu addrs and
> dma addrs. It subsumes the role of "dev->dma_pfn_offset" which was only
> capable of holding a single uniform offset and had no region bounds
> checking.
>
> The function of_dma_get_range() has been modified so that it takes a single
> argument -- the device node -- and returns a map, NULL, or an error code.
> The map is an array that holds the information regarding the DMA regions.
> Each range entry contains the address offset, the cpu_start address, the
> dma_start address, and the size of the region.
>
> of_dma_configure() is the typical manner to set range offsets but there are
> a number of ad hoc assignments to "dev->dma_pfn_offset" in the kernel
> driver code. These cases now invoke the function
> dma_attach_offset_range(dev, cpu_addr, dma_addr, size).
>
> Signed-off-by: Jim Quinlan
> ---
[...]
> diff --git a/drivers/of/address.c b/drivers/of/address.c
> index 8eea3f6e29a4..4b718d199efe 100644
> --- a/drivers/of/address.c
> +++ b/drivers/of/address.c
> @@ -918,33 +918,33 @@ void __iomem *of_io_request_and_map(struct device_node
> *np, int index,
> }
> EXPORT_SYMBOL(of_io_request_and_map);
>
> +#ifdef CONFIG_HAS_DMA
> /**
> - * of_dma_get_range - Get DMA range info
> + * of_dma_get_range - Get DMA range info and put it into a map array
> * @np: device node to get DMA range info
> - * @dma_addr:pointer to store initial DMA address of DMA range
> - * @paddr: pointer to store initial CPU address of DMA range
> - * @size:pointer to store size of DMA range
> + * @map: dma range structure to return
> *
> * Look in bottom up direction for the first "dma-ranges" property
> - * and parse it.
> - * dma-ranges format:
> + * and parse it. Put the information into a DMA offset map array.
> + *
> + * dma-ranges format:
> * DMA addr (dma_addr) : naddr cells
> * CPU addr (phys_addr_t) : pna cells
> * size: nsize cells
> *
> - * It returns -ENODEV if "dma-ranges" property was not found
> - * for this device in DT.
> + * It returns -ENODEV if "dma-ranges" property was not found for this
> + * device in the DT.
> */
> -int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64
> *size)
> +int of_dma_get_range(struct device_node *np, const struct bus_dma_region
> **map)
> {
> struct device_node *node = of_node_get(np);
> const __be32 *ranges = NULL;
> - int len;
> - int ret = 0;
> bool found_dma_ranges = false;
> struct of_range_parser parser;
> struct of_range range;
> - u64 dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
> + struct bus_dma_region *r;
> + int len, num_ranges = 0;
> + int ret;
>
> while (node) {
> ranges = of_get_property(node, "dma-ranges", );
> @@ -970,44 +970,35 @@ int of_dma_get_range(struct device_node *np, u64
> *dma_addr, u64 *paddr, u64 *siz
> }
>
> of_dma_range_parser_init(, node);
> + for_each_of_range(, )
> + num_ranges++;
> +
> + of_dma_range_parser_init(, node);
> +
> + ret = -ENOMEM;
> + r = kcalloc(num_ranges + 1, sizeof(*r), GFP_KERNEL);
> + if (!r)
> + goto out;
>
> + /*
> + * Record all info in the generic DMA ranges array for struct device.
> + */
> + *map = r;
> for_each_of_range(, ) {
> pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
>range.bus_addr, range.cpu_addr, range.size);
> -
> - if (dma_offset && range.cpu_addr - range.bus_addr !=
> dma_offset) {
> - pr_warn("Can't handle multiple dma-ranges with
> different offsets on node(%pOF)\n", node);
> - /* Don't error out as we'd break some existing DTs */
> - continue;
> - }
> - dma_offset = range.cpu_addr - range.bus_addr;
> -
> - /* Take lower and upper limits */
> - if (range.bus_addr < dma_start)
> - dma_start = range.bus_addr;
> - if (range.bus_addr + range.size > dma_end)
> - dma_end = range.bus_addr + range.size;
> - }
> -
> - if (dma_start >= dma_end) {
> - ret = -EINVAL;
> - pr_debug("Invalid DMA ranges configuration on node(%pOF)\n",
> - node);
> - goto out;
> + r->cpu_start = range.cpu_addr;
> + r->dma_start = range.bus_addr;
> + r->size = range.size;
> + r->offset = (u64)range.cpu_addr - (u64)range.bus_addr;
> + r++;
> }
>
> - *dma_addr = dma_start;
> - *size = dma_end - dma_start;
> - *paddr = dma_start + dma_offset;
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Thu, Jul 30, 2020 at 10:44 AM Jim Quinlan wrote: > > On Wed, Jul 29, 2020 at 10:28 AM Rob Herring wrote: > > > > On Wed, Jul 29, 2020 at 12:19 AM Christoph Hellwig wrote: > > > > > > On Tue, Jul 28, 2020 at 02:24:51PM -0400, Jim Quinlan wrote: > > > > I started using devm_kcalloc() but at least two reviewers convinced me > > > > to just use kcalloc(). In addition, when I was using devm_kcalloc() > > > > it was awkward because 'dev' is not available to this function. > > > > > > > > It comes down to whether unbind/binding the device N times is actually > > > > a reasonable usage. As for my experience I've seen two cases: (1) my > > > > overnight "bind/unbind the PCIe RC driver" script, and we have a > > > > customer who does an unbind/bind as a hail mary to bring back life to > > > > their dead EP device. If the latter case happens repeatedly, there > > > > are bigger problems. > > > > > > We can't just leak the allocations. Do you have a pointer to the > > > arguments against managed resources? I'm generally not a huge fan > > > of the managed resources, but for a case like this they actually seem > > > useful. If we don't use the managed resources we'll at leat need > > > to explicitly free the resources when freeing the device. > > > > The lifetime for devm_kcalloc may not be what we want here. devm > > allocs are freed on probe fail or remove, not on freeing the device > > (there is a just in case free there too though). > > What do you suggest doing as an alternative? I believe I gave it already. Put a kfree in the struct device release function. But you can't be passing the pointer from device to device if you go that route. You'd have to either copy the struct or check if it's the same as the parent's and skip the kfree in that case. Rob ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Thu, 2020-07-30 at 13:25 -0400, Jim Quinlan wrote:
> On Thu, Jul 30, 2020 at 1:05 PM Nicolas Saenz Julienne
> wrote:
> > Hi Jim,
> >
> > On Fri, 2020-07-24 at 16:33 -0400, Jim Quinlan wrote:
> > > static void __init of_unittest_pci_dma_ranges(void)
> > > diff --git a/drivers/pci/controller/pcie-brcmstb.c
> > > b/drivers/pci/controller/pcie-brcmstb.c
> > > index bfc2542d54a8..8dacb9d3b7b6 100644
> > > --- a/drivers/pci/controller/pcie-brcmstb.c
> > > +++ b/drivers/pci/controller/pcie-brcmstb.c
> > > @@ -1197,6 +1197,7 @@ static int brcm_pcie_probe(struct platform_device
> > > *pdev)
> > > ret = brcm_phy_start(pcie);
> > > if (ret) {
> > > dev_err(pcie->dev, "failed to start phy\n");
> > > + reset_control_assert(pcie->rescal);
> > > return ret;
> > > }
> >
> > I think this sneaked in from another patch.
> Thanks Nicolas. BTW, at some point will you be able to test my
> patchset on the RP4 to see if I broke anything?
Yes of course, I'll have a go at it tomorrow.
Regards,
Nicolas
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Thu, Jul 30, 2020 at 1:05 PM Nicolas Saenz Julienne
wrote:
>
> Hi Jim,
>
> On Fri, 2020-07-24 at 16:33 -0400, Jim Quinlan wrote:
> > static void __init of_unittest_pci_dma_ranges(void)
> > diff --git a/drivers/pci/controller/pcie-brcmstb.c
> > b/drivers/pci/controller/pcie-brcmstb.c
> > index bfc2542d54a8..8dacb9d3b7b6 100644
> > --- a/drivers/pci/controller/pcie-brcmstb.c
> > +++ b/drivers/pci/controller/pcie-brcmstb.c
> > @@ -1197,6 +1197,7 @@ static int brcm_pcie_probe(struct platform_device
> > *pdev)
> > ret = brcm_phy_start(pcie);
> > if (ret) {
> > dev_err(pcie->dev, "failed to start phy\n");
> > + reset_control_assert(pcie->rescal);
> > return ret;
> > }
>
> I think this sneaked in from another patch.
Thanks Nicolas. BTW, at some point will you be able to test my
patchset on the RP4 to see if I broke anything?
Thanks again,
Jim
>
> Regards,
> Nicolas
>
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
Hi Jim,
On Fri, 2020-07-24 at 16:33 -0400, Jim Quinlan wrote:
> static void __init of_unittest_pci_dma_ranges(void)
> diff --git a/drivers/pci/controller/pcie-brcmstb.c
> b/drivers/pci/controller/pcie-brcmstb.c
> index bfc2542d54a8..8dacb9d3b7b6 100644
> --- a/drivers/pci/controller/pcie-brcmstb.c
> +++ b/drivers/pci/controller/pcie-brcmstb.c
> @@ -1197,6 +1197,7 @@ static int brcm_pcie_probe(struct platform_device *pdev)
> ret = brcm_phy_start(pcie);
> if (ret) {
> dev_err(pcie->dev, "failed to start phy\n");
> + reset_control_assert(pcie->rescal);
> return ret;
> }
I think this sneaked in from another patch.
Regards,
Nicolas
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Thu, 2020-07-30 at 12:44 -0400, Jim Quinlan wrote: > On Wed, Jul 29, 2020 at 10:28 AM Rob Herring wrote: > > On Wed, Jul 29, 2020 at 12:19 AM Christoph Hellwig wrote: > > > On Tue, Jul 28, 2020 at 02:24:51PM -0400, Jim Quinlan wrote: > > > > I started using devm_kcalloc() but at least two reviewers convinced me > > > > to just use kcalloc(). In addition, when I was using devm_kcalloc() > > > > it was awkward because 'dev' is not available to this function. > > > > > > > > It comes down to whether unbind/binding the device N times is actually > > > > a reasonable usage. As for my experience I've seen two cases: (1) my > > > > overnight "bind/unbind the PCIe RC driver" script, and we have a > > > > customer who does an unbind/bind as a hail mary to bring back life to > > > > their dead EP device. If the latter case happens repeatedly, there > > > > are bigger problems. > > > > > > We can't just leak the allocations. Do you have a pointer to the > > > arguments against managed resources? I'm generally not a huge fan > > > of the managed resources, but for a case like this they actually seem > > > useful. If we don't use the managed resources we'll at leat need > > > to explicitly free the resources when freeing the device. > > > > The lifetime for devm_kcalloc may not be what we want here. devm > > allocs are freed on probe fail or remove, not on freeing the device > > (there is a just in case free there too though). > > What do you suggest doing as an alternative? I haven't given the idea much thought, but how about using a kref in the first element of your bus_dma_region array. It should be increased upon assigning it to a device and decreased it upon destroying it (triggering the free once it hits 0). It would take care of this memory leak, but also useful where you're sharing bus_dma_regions between devices. Regards, Nicolas ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Wed, Jul 29, 2020 at 10:28 AM Rob Herring wrote: > > On Wed, Jul 29, 2020 at 12:19 AM Christoph Hellwig wrote: > > > > On Tue, Jul 28, 2020 at 02:24:51PM -0400, Jim Quinlan wrote: > > > I started using devm_kcalloc() but at least two reviewers convinced me > > > to just use kcalloc(). In addition, when I was using devm_kcalloc() > > > it was awkward because 'dev' is not available to this function. > > > > > > It comes down to whether unbind/binding the device N times is actually > > > a reasonable usage. As for my experience I've seen two cases: (1) my > > > overnight "bind/unbind the PCIe RC driver" script, and we have a > > > customer who does an unbind/bind as a hail mary to bring back life to > > > their dead EP device. If the latter case happens repeatedly, there > > > are bigger problems. > > > > We can't just leak the allocations. Do you have a pointer to the > > arguments against managed resources? I'm generally not a huge fan > > of the managed resources, but for a case like this they actually seem > > useful. If we don't use the managed resources we'll at leat need > > to explicitly free the resources when freeing the device. > > The lifetime for devm_kcalloc may not be what we want here. devm > allocs are freed on probe fail or remove, not on freeing the device > (there is a just in case free there too though). What do you suggest doing as an alternative? Jim > > > Rob ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Wed, Jul 29, 2020 at 12:19 AM Christoph Hellwig wrote: > > On Tue, Jul 28, 2020 at 02:24:51PM -0400, Jim Quinlan wrote: > > I started using devm_kcalloc() but at least two reviewers convinced me > > to just use kcalloc(). In addition, when I was using devm_kcalloc() > > it was awkward because 'dev' is not available to this function. > > > > It comes down to whether unbind/binding the device N times is actually > > a reasonable usage. As for my experience I've seen two cases: (1) my > > overnight "bind/unbind the PCIe RC driver" script, and we have a > > customer who does an unbind/bind as a hail mary to bring back life to > > their dead EP device. If the latter case happens repeatedly, there > > are bigger problems. > > We can't just leak the allocations. Do you have a pointer to the > arguments against managed resources? I'm generally not a huge fan > of the managed resources, but for a case like this they actually seem > useful. If we don't use the managed resources we'll at leat need > to explicitly free the resources when freeing the device. The lifetime for devm_kcalloc may not be what we want here. devm allocs are freed on probe fail or remove, not on freeing the device (there is a just in case free there too though). Rob ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Wed, Jul 29, 2020 at 2:19 AM Christoph Hellwig wrote: > > On Tue, Jul 28, 2020 at 02:24:51PM -0400, Jim Quinlan wrote: > > I started using devm_kcalloc() but at least two reviewers convinced me > > to just use kcalloc(). In addition, when I was using devm_kcalloc() > > it was awkward because 'dev' is not available to this function. > > > > It comes down to whether unbind/binding the device N times is actually > > a reasonable usage. As for my experience I've seen two cases: (1) my > > overnight "bind/unbind the PCIe RC driver" script, and we have a > > customer who does an unbind/bind as a hail mary to bring back life to > > their dead EP device. If the latter case happens repeatedly, there > > are bigger problems. > > We can't just leak the allocations. Do you have a pointer to the > arguments against managed resources? I'm generally not a huge fan > of the managed resources, but for a case like this they actually seem > useful. If we don't use the managed resources we'll at leat need > to explicitly free the resources when freeing the device. Actually, there were no arguments for using an unmanaged kcalloc, just comments [1], [2]. When it was rightly suggested to have 'dev' dropped from of_dma_range() [3], I submitted v6 to accomplish this. But now of_dma_range() would call kcalloc(), and then of_dma_configure() was required to "dup" the result, requiring a devm_kcalloc(), memcpy(), and kfree(). This was awkward, so considering [1], [2], I dropped the devm_kcalloc() and submitted v7. So I can easily revert to the awkward code of v6. But I'm hoping you have a more elegant solution :-) Thanks, Jim [1] [v6, Andy Shevchenko] > + /* We want the offset map to be device-managed, so alloc & copy > */ > + dev->dma_range_map = devm_kcalloc(dev, num_ranges + 1, > sizeof(*r), > + GFP_KERNEL); The question is how many times per device lifetime this can be called? [2] [v2, Andy Shevchenko] > + r = devm_kzalloc(dev, r_size, GFP_KERNEL); devm_?! Looking at r_size it should be rather kcalloc(). [3] [v3, Nicolas Saenz Julienne] > I agree with you. The reason I needed a "struct device *" in the call is > because I wanted to make sure the memory that is alloc'd belongs to the > device that needs it. If I do a regular kzalloc(), this memory will become > a leak once someone starts unbinding/binding their device. Also, in all > uses of of_dma_rtange() -- there is only one -- a dev is required as one > can't attach an offset map to NULL. > > I do see that there are a number of functions in drivers/of/*.c that > take 'struct device *dev' as an argument so there is precedent for > something like this. Regardless, I need an owner to the memory I > alloc(). I understand the need for dev to be around, devm_*() is key. But also it's important to keep the functions on purpose. And if of_dma_get_range() starts setting ranges it calls, for the very least, for a function rename. Although I'd rather split the parsing and setting of ranges as mentioned earlier. That said, I get that's a more drastic move. ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Tue, Jul 28, 2020 at 02:24:51PM -0400, Jim Quinlan wrote: > I started using devm_kcalloc() but at least two reviewers convinced me > to just use kcalloc(). In addition, when I was using devm_kcalloc() > it was awkward because 'dev' is not available to this function. > > It comes down to whether unbind/binding the device N times is actually > a reasonable usage. As for my experience I've seen two cases: (1) my > overnight "bind/unbind the PCIe RC driver" script, and we have a > customer who does an unbind/bind as a hail mary to bring back life to > their dead EP device. If the latter case happens repeatedly, there > are bigger problems. We can't just leak the allocations. Do you have a pointer to the arguments against managed resources? I'm generally not a huge fan of the managed resources, but for a case like this they actually seem useful. If we don't use the managed resources we'll at leat need to explicitly free the resources when freeing the device. ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
Hi Christoph,
On Tue, Jul 28, 2020 at 8:33 AM Christoph Hellwig wrote:
>
> A few tiny nitpicks:
>
> The subject should have the dma-mapping prefix, this doesn't
> really touch the device core.
>
> > - rc = of_dma_get_range(np, _addr, , );
> > + rc = of_dma_get_range(np, );
> > + rc = PTR_ERR_OR_ZERO(map);
>
> I don't think you need the PTR_ERR_OR_ZERO line here, of_dma_get_range
> returns the error.
Yes, that link needs to be deleted.
>
> > +int dma_set_offset_range(struct device *dev, phys_addr_t cpu_start,
> > + dma_addr_t dma_start, u64 size)hh
> > +{
> > + struct bus_dma_region *map;
> > + u64 offset = (u64)cpu_start - (u64)dma_start;
> > +
> > + if (!dev)
> > + return -ENODEV;
>
> I don't think we need the NULL protection here, all DMA API calls
> expect a device.
Yes, your review-patch removed it but left the comment about the
function returning -ENODEV. So I wasn't sure to leave it in or not.
>
> > + if (!offset)
> > + return 0;
> > +
> > + /*
> > + * See if a map already exists and we already encompass the new range:
> > + */
> > + if (dev->dma_range_map) {
> > + if (dma_range_overlaps(dev, cpu_start, dma_start, size,
> > offset))
> > + return 0;
> > + dev_err(dev, "attempt to add conflicting DMA range to
> > existing map\n");
> > + return -EINVAL;
> > + }
>
> And here why do we need the overlap check at all? I'd be tempted to
> always return an error for this case.
I believe the overlap check was your suggestion or at least in your
review-patch? I'm fine with just returning an error.
>
> What is the plan to merge this? Do you want all this to go into one
> tree, or get as many bits into the applicable trees for 5.9 and then
> finish up for 5.10? If the former I can apply it to the dma-mapping
> tree and just fix up the nitpicks.
Whatever you think is best -- I would be quite happy if you could
accept at least the "dma_range_map" commit. Of course I'd be most
happy if the entire patchset were accepted, but perhaps you can just
apply the "dma_range_map" commit, and I will continue to bang away at
getting the N-1 PCIe-related commits ack'd and accepted.
Thanks much!
Jim Quinlan
Broadcom STB
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Tue, Jul 28, 2020 at 11:05 AM Rob Herring wrote:
>
> On Fri, Jul 24, 2020 at 2:45 PM Jim Quinlan
> wrote:
> >
> > The new field 'dma_range_map' in struct device is used to facilitate the
> > use of single or multiple offsets between mapping regions of cpu addrs and
> > dma addrs. It subsumes the role of "dev->dma_pfn_offset" which was only
> > capable of holding a single uniform offset and had no region bounds
> > checking.
> >
> > The function of_dma_get_range() has been modified so that it takes a single
> > argument -- the device node -- and returns a map, NULL, or an error code.
> > The map is an array that holds the information regarding the DMA regions.
> > Each range entry contains the address offset, the cpu_start address, the
> > dma_start address, and the size of the region.
> >
> > of_dma_configure() is the typical manner to set range offsets but there are
> > a number of ad hoc assignments to "dev->dma_pfn_offset" in the kernel
> > driver code. These cases now invoke the function
> > dma_attach_offset_range(dev, cpu_addr, dma_addr, size).
> >
> > Signed-off-by: Jim Quinlan
> > ---
>
> [...]
>
> > diff --git a/drivers/of/address.c b/drivers/of/address.c
> > index 8eea3f6e29a4..4b718d199efe 100644
> > --- a/drivers/of/address.c
> > +++ b/drivers/of/address.c
> > @@ -918,33 +918,33 @@ void __iomem *of_io_request_and_map(struct
> > device_node *np, int index,
> > }
> > EXPORT_SYMBOL(of_io_request_and_map);
> >
> > +#ifdef CONFIG_HAS_DMA
> > /**
> > - * of_dma_get_range - Get DMA range info
> > + * of_dma_get_range - Get DMA range info and put it into a map array
> > * @np:device node to get DMA range info
> > - * @dma_addr: pointer to store initial DMA address of DMA range
> > - * @paddr: pointer to store initial CPU address of DMA range
> > - * @size: pointer to store size of DMA range
> > + * @map: dma range structure to return
> > *
> > * Look in bottom up direction for the first "dma-ranges" property
> > - * and parse it.
> > - * dma-ranges format:
> > + * and parse it. Put the information into a DMA offset map array.
> > + *
> > + * dma-ranges format:
> > * DMA addr (dma_addr) : naddr cells
> > * CPU addr (phys_addr_t) : pna cells
> > * size: nsize cells
> > *
> > - * It returns -ENODEV if "dma-ranges" property was not found
> > - * for this device in DT.
> > + * It returns -ENODEV if "dma-ranges" property was not found for this
> > + * device in the DT.
> > */
> > -int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr,
> > u64 *size)
> > +int of_dma_get_range(struct device_node *np, const struct bus_dma_region
> > **map)
> > {
> > struct device_node *node = of_node_get(np);
> > const __be32 *ranges = NULL;
> > - int len;
> > - int ret = 0;
> > bool found_dma_ranges = false;
> > struct of_range_parser parser;
> > struct of_range range;
> > - u64 dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
> > + struct bus_dma_region *r;
> > + int len, num_ranges = 0;
> > + int ret;
> >
> > while (node) {
> > ranges = of_get_property(node, "dma-ranges", );
> > @@ -970,44 +970,35 @@ int of_dma_get_range(struct device_node *np, u64
> > *dma_addr, u64 *paddr, u64 *siz
> > }
> >
> > of_dma_range_parser_init(, node);
> > + for_each_of_range(, )
> > + num_ranges++;
> > +
> > + of_dma_range_parser_init(, node);
> > +
> > + ret = -ENOMEM;
> > + r = kcalloc(num_ranges + 1, sizeof(*r), GFP_KERNEL);
> > + if (!r)
> > + goto out;
>
> AFAICT, you have the error cases covered, but you are leaking memory
> if the device is removed.
Hi Rob,
I started using devm_kcalloc() but at least two reviewers convinced me
to just use kcalloc(). In addition, when I was using devm_kcalloc()
it was awkward because 'dev' is not available to this function.
It comes down to whether unbind/binding the device N times is actually
a reasonable usage. As for my experience I've seen two cases: (1) my
overnight "bind/unbind the PCIe RC driver" script, and we have a
customer who does an unbind/bind as a hail mary to bring back life to
their dead EP device. If the latter case happens repeatedly, there
are bigger problems.
>
>
> [...]
>b
> > diff --git a/drivers/remoteproc/remoteproc_core.c
> > b/drivers/remoteproc/remoteproc_core.c
> > index 9f04c30c4aaf..49242dd6176e 100644
> > --- a/drivers/remoteproc/remoteproc_core.c
> > +++ b/drivers/remoteproc/remoteproc_core.c
> > @@ -519,7 +519,7 @@ static int rproc_handle_vdev(struct rproc *rproc,
> > struct fw_rsc_vdev *rsc,
> > /* Initialise vdev subdevice */
> > snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index);
> > rvdev->dev.parent = >dev;
> > - rvdev->dev.dma_pfn_offset = rproc->dev.parent->dma_pfn_offset;
> > + rvdev->dev.dma_range_map =
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
On Fri, Jul 24, 2020 at 2:45 PM Jim Quinlan wrote:
>
> The new field 'dma_range_map' in struct device is used to facilitate the
> use of single or multiple offsets between mapping regions of cpu addrs and
> dma addrs. It subsumes the role of "dev->dma_pfn_offset" which was only
> capable of holding a single uniform offset and had no region bounds
> checking.
>
> The function of_dma_get_range() has been modified so that it takes a single
> argument -- the device node -- and returns a map, NULL, or an error code.
> The map is an array that holds the information regarding the DMA regions.
> Each range entry contains the address offset, the cpu_start address, the
> dma_start address, and the size of the region.
>
> of_dma_configure() is the typical manner to set range offsets but there are
> a number of ad hoc assignments to "dev->dma_pfn_offset" in the kernel
> driver code. These cases now invoke the function
> dma_attach_offset_range(dev, cpu_addr, dma_addr, size).
>
> Signed-off-by: Jim Quinlan
> ---
[...]
> diff --git a/drivers/of/address.c b/drivers/of/address.c
> index 8eea3f6e29a4..4b718d199efe 100644
> --- a/drivers/of/address.c
> +++ b/drivers/of/address.c
> @@ -918,33 +918,33 @@ void __iomem *of_io_request_and_map(struct device_node
> *np, int index,
> }
> EXPORT_SYMBOL(of_io_request_and_map);
>
> +#ifdef CONFIG_HAS_DMA
> /**
> - * of_dma_get_range - Get DMA range info
> + * of_dma_get_range - Get DMA range info and put it into a map array
> * @np:device node to get DMA range info
> - * @dma_addr: pointer to store initial DMA address of DMA range
> - * @paddr: pointer to store initial CPU address of DMA range
> - * @size: pointer to store size of DMA range
> + * @map: dma range structure to return
> *
> * Look in bottom up direction for the first "dma-ranges" property
> - * and parse it.
> - * dma-ranges format:
> + * and parse it. Put the information into a DMA offset map array.
> + *
> + * dma-ranges format:
> * DMA addr (dma_addr) : naddr cells
> * CPU addr (phys_addr_t) : pna cells
> * size: nsize cells
> *
> - * It returns -ENODEV if "dma-ranges" property was not found
> - * for this device in DT.
> + * It returns -ENODEV if "dma-ranges" property was not found for this
> + * device in the DT.
> */
> -int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64
> *size)
> +int of_dma_get_range(struct device_node *np, const struct bus_dma_region
> **map)
> {
> struct device_node *node = of_node_get(np);
> const __be32 *ranges = NULL;
> - int len;
> - int ret = 0;
> bool found_dma_ranges = false;
> struct of_range_parser parser;
> struct of_range range;
> - u64 dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
> + struct bus_dma_region *r;
> + int len, num_ranges = 0;
> + int ret;
>
> while (node) {
> ranges = of_get_property(node, "dma-ranges", );
> @@ -970,44 +970,35 @@ int of_dma_get_range(struct device_node *np, u64
> *dma_addr, u64 *paddr, u64 *siz
> }
>
> of_dma_range_parser_init(, node);
> + for_each_of_range(, )
> + num_ranges++;
> +
> + of_dma_range_parser_init(, node);
> +
> + ret = -ENOMEM;
> + r = kcalloc(num_ranges + 1, sizeof(*r), GFP_KERNEL);
> + if (!r)
> + goto out;
AFAICT, you have the error cases covered, but you are leaking memory
if the device is removed.
[...]
> diff --git a/drivers/remoteproc/remoteproc_core.c
> b/drivers/remoteproc/remoteproc_core.c
> index 9f04c30c4aaf..49242dd6176e 100644
> --- a/drivers/remoteproc/remoteproc_core.c
> +++ b/drivers/remoteproc/remoteproc_core.c
> @@ -519,7 +519,7 @@ static int rproc_handle_vdev(struct rproc *rproc, struct
> fw_rsc_vdev *rsc,
> /* Initialise vdev subdevice */
> snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index);
> rvdev->dev.parent = >dev;
> - rvdev->dev.dma_pfn_offset = rproc->dev.parent->dma_pfn_offset;
> + rvdev->dev.dma_range_map = rproc->dev.parent->dma_range_map;
But doing this means you can't just free the dma_range_map. You need
to do a copy here or you'd have to refcount it. Or I suppose you could
check if it the child has a different dma_range_map ptr than the
parent.
Rob
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
A few tiny nitpicks:
The subject should have the dma-mapping prefix, this doesn't
really touch the device core.
> - rc = of_dma_get_range(np, _addr, , );
> + rc = of_dma_get_range(np, );
> + rc = PTR_ERR_OR_ZERO(map);
I don't think you need the PTR_ERR_OR_ZERO line here, of_dma_get_range
returns the error.
> +int dma_set_offset_range(struct device *dev, phys_addr_t cpu_start,
> + dma_addr_t dma_start, u64 size)
> +{
> + struct bus_dma_region *map;
> + u64 offset = (u64)cpu_start - (u64)dma_start;
> +
> + if (!dev)
> + return -ENODEV;
I don't think we need the NULL protection here, all DMA API calls
expect a device.
> + if (!offset)
> + return 0;
> +
> + /*
> + * See if a map already exists and we already encompass the new range:
> + */
> + if (dev->dma_range_map) {
> + if (dma_range_overlaps(dev, cpu_start, dma_start, size, offset))
> + return 0;
> + dev_err(dev, "attempt to add conflicting DMA range to existing
> map\n");
> + return -EINVAL;
> + }
And here why do we need the overlap check at all? I'd be tempted to
always return an error for this case.
What is the plan to merge this? Do you want all this to go into one
tree, or get as many bits into the applicable trees for 5.9 and then
finish up for 5.10? If the former I can apply it to the dma-mapping
tree and just fix up the nitpicks.
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 08/12] device core: Introduce DMA range map, supplanting dma_pfn_offset
The new field 'dma_range_map' in struct device is used to facilitate the
use of single or multiple offsets between mapping regions of cpu addrs and
dma addrs. It subsumes the role of "dev->dma_pfn_offset" which was only
capable of holding a single uniform offset and had no region bounds
checking.
The function of_dma_get_range() has been modified so that it takes a single
argument -- the device node -- and returns a map, NULL, or an error code.
The map is an array that holds the information regarding the DMA regions.
Each range entry contains the address offset, the cpu_start address, the
dma_start address, and the size of the region.
of_dma_configure() is the typical manner to set range offsets but there are
a number of ad hoc assignments to "dev->dma_pfn_offset" in the kernel
driver code. These cases now invoke the function
dma_attach_offset_range(dev, cpu_addr, dma_addr, size).
Signed-off-by: Jim Quinlan
---
arch/arm/include/asm/dma-mapping.h| 10 +--
arch/arm/mach-keystone/keystone.c | 17 +++--
arch/sh/drivers/pci/pcie-sh7786.c | 9 +--
arch/sh/kernel/dma-coherent.c | 15 ++--
arch/x86/pci/sta2x11-fixup.c | 7 +-
drivers/acpi/arm64/iort.c | 5 +-
drivers/gpu/drm/sun4i/sun4i_backend.c | 5 +-
drivers/iommu/io-pgtable-arm.c| 2 +-
.../platform/sunxi/sun4i-csi/sun4i_csi.c | 5 +-
.../platform/sunxi/sun6i-csi/sun6i_csi.c | 4 +-
drivers/of/address.c | 71 ---
drivers/of/device.c | 43 ++-
drivers/of/of_private.h | 10 +--
drivers/of/unittest.c | 32 ++---
drivers/pci/controller/pcie-brcmstb.c | 1 +
drivers/remoteproc/remoteproc_core.c | 2 +-
.../staging/media/sunxi/cedrus/cedrus_hw.c| 7 +-
drivers/usb/core/message.c| 4 +-
drivers/usb/core/usb.c| 2 +-
include/linux/device.h| 4 +-
include/linux/dma-direct.h| 8 +--
include/linux/dma-mapping.h | 34 +
kernel/dma/coherent.c | 10 +--
kernel/dma/mapping.c | 63
24 files changed, 248 insertions(+), 122 deletions(-)
diff --git a/arch/arm/include/asm/dma-mapping.h
b/arch/arm/include/asm/dma-mapping.h
index bdd80ddbca34..2405afeb7957 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -35,8 +35,11 @@ static inline const struct dma_map_ops
*get_arch_dma_ops(struct bus_type *bus)
#ifndef __arch_pfn_to_dma
static inline dma_addr_t pfn_to_dma(struct device *dev, unsigned long pfn)
{
- if (dev)
- pfn -= dev->dma_pfn_offset;
+ if (dev) {
+ phys_addr_t paddr = PFN_PHYS(pfn);
+
+ pfn -= (dma_offset_from_phys_addr(dev, paddr) >> PAGE_SHIFT);
+ }
return (dma_addr_t)__pfn_to_bus(pfn);
}
@@ -45,8 +48,7 @@ static inline unsigned long dma_to_pfn(struct device *dev,
dma_addr_t addr)
unsigned long pfn = __bus_to_pfn(addr);
if (dev)
- pfn += dev->dma_pfn_offset;
-
+ pfn += (dma_offset_from_dma_addr(dev, addr) >> PAGE_SHIFT);
return pfn;
}
diff --git a/arch/arm/mach-keystone/keystone.c
b/arch/arm/mach-keystone/keystone.c
index 638808c4e122..78808942ad1c 100644
--- a/arch/arm/mach-keystone/keystone.c
+++ b/arch/arm/mach-keystone/keystone.c
@@ -8,6 +8,7 @@
*/
#include
#include
+#include
#include
#include
#include
@@ -24,8 +25,6 @@
#include "keystone.h"
-static unsigned long keystone_dma_pfn_offset __read_mostly;
-
static int keystone_platform_notifier(struct notifier_block *nb,
unsigned long event, void *data)
{
@@ -38,9 +37,12 @@ static int keystone_platform_notifier(struct notifier_block
*nb,
return NOTIFY_BAD;
if (!dev->of_node) {
- dev->dma_pfn_offset = keystone_dma_pfn_offset;
- dev_err(dev, "set dma_pfn_offset%08lx\n",
- dev->dma_pfn_offset);
+ int ret = dma_set_offset_range(dev, KEYSTONE_HIGH_PHYS_START,
+ KEYSTONE_LOW_PHYS_START,
+ KEYSTONE_HIGH_PHYS_SIZE);
+ dev_err(dev, "set dma_offset%08llx%s\n",
+ KEYSTONE_HIGH_PHYS_START - KEYSTONE_LOW_PHYS_START,
+ ret ? " failed" : "");
}
return NOTIFY_OK;
}
@@ -51,11 +53,8 @@ static struct notifier_block platform_nb = {
static void __init keystone_init(void)
{
- if (PHYS_OFFSET >= KEYSTONE_HIGH_PHYS_START) {
- keystone_dma_pfn_offset = PFN_DOWN(KEYSTONE_HIGH_PHYS_START -
- KEYSTONE_LOW_PHYS_START);
+ if
