Re: [PATCH kernel v5 14/20] powerpc/powernv/npu: Add compound IOMMU groups
Alexey Kardashevskiy writes:
> On 19/12/2018 21:00, Michael Ellerman wrote:
>> Alexey Kardashevskiy writes:
>>> On 19/12/2018 11:17, Michael Ellerman wrote:
Alexey Kardashevskiy writes:
> diff --git a/arch/powerpc/platforms/powernv/npu-dma.c
> b/arch/powerpc/platforms/powernv/npu-dma.c
> index dc629ee..3468eaa 100644
> --- a/arch/powerpc/platforms/powernv/npu-dma.c
> +++ b/arch/powerpc/platforms/powernv/npu-dma.c
> @@ -372,8 +358,263 @@ struct npu {
...
> +
> +static void pnv_comp_attach_table_group(struct npu_comp *npucomp,
> + struct pnv_ioda_pe *pe)
> +{
> + if (WARN_ON(npucomp->pe_num == NV_NPU_MAX_PE_NUM))
> + return;
> +
> + npucomp->pe[npucomp->pe_num] = pe;
> + ++npucomp->pe_num;
> +}
> +
> +struct iommu_table_group *pnv_try_setup_npu_table_group(struct
> pnv_ioda_pe *pe)
> +{
> + struct iommu_table_group *table_group;
> + struct npu_comp *npucomp;
> + struct pci_dev *gpdev = NULL;
> + struct pci_controller *hose;
> + struct pci_dev *npdev;
> +
> + list_for_each_entry(gpdev, >pbus->devices, bus_list) {
> + npdev = pnv_pci_get_npu_dev(gpdev, 0);
> + if (npdev)
> + break;
> + }
> +
> + if (!npdev)
> + /* It is not an NPU attached device, skip */
> + return NULL;
This breaks some configs with:
arch/powerpc/platforms/powernv/npu-dma.c:550:5: error: 'npdev' may be
used uninitialized in this function [-Werror=uninitialized]
>>>
>>>
>>> gcc 5, 7 and 8 do not warn about this, I have to disable
>>> list_for_each_entry() above to recreate this.
>>>
>>> I even compiled gcc 5.5 which some of your buildmachines use and yet no
>>> error on this:
>>>
>>> make O=/home/aik/pbuild/kernel-le/ KCFLAGS=-Werror=all ARCH=powerpc
>>> CROSS_COMPILE=/opt/cross/gcc-powerpc64le-linux-5.5.0-nolibc/bin/powerpc64le-linux-
>>> arch/powerpc/platforms/powernv/npu-dma.o
>>
>> Odd. That error is from kisskb like the others.
>>
>> http://kisskb.ellerman.id.au/kisskb/buildresult/13622793/
>>
>> Seems it's GCC 4.6.3 that is producing that one. Not sure why newer
>> compilers aren't warning about it.
>>
>>
>> It's pretty obviously correct though, unless you can prove that the list
>> is never empty?
>
> I know these are correct and I want to catch these before I post patches :-/
>
> Can I get that gcc 4.6.3? It does not compile on my ubuntu for whatever
> reason.
It's the old korg one, it's on the ka's in /opt/cross.
It's this one:
https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/x86_64-gcc-4.6.3-nolibc_powerpc64-linux.tar.xz
cheers
Re: [PATCH kernel v5 14/20] powerpc/powernv/npu: Add compound IOMMU groups
On 19/12/2018 21:00, Michael Ellerman wrote:
> Alexey Kardashevskiy writes:
>> On 19/12/2018 11:17, Michael Ellerman wrote:
>>> Alexey Kardashevskiy writes:
diff --git a/arch/powerpc/platforms/powernv/npu-dma.c
b/arch/powerpc/platforms/powernv/npu-dma.c
index dc629ee..3468eaa 100644
--- a/arch/powerpc/platforms/powernv/npu-dma.c
+++ b/arch/powerpc/platforms/powernv/npu-dma.c
@@ -372,8 +358,263 @@ struct npu {
>>> ...
+
+static void pnv_comp_attach_table_group(struct npu_comp *npucomp,
+ struct pnv_ioda_pe *pe)
+{
+ if (WARN_ON(npucomp->pe_num == NV_NPU_MAX_PE_NUM))
+ return;
+
+ npucomp->pe[npucomp->pe_num] = pe;
+ ++npucomp->pe_num;
+}
+
+struct iommu_table_group *pnv_try_setup_npu_table_group(struct
pnv_ioda_pe *pe)
+{
+ struct iommu_table_group *table_group;
+ struct npu_comp *npucomp;
+ struct pci_dev *gpdev = NULL;
+ struct pci_controller *hose;
+ struct pci_dev *npdev;
+
+ list_for_each_entry(gpdev, >pbus->devices, bus_list) {
+ npdev = pnv_pci_get_npu_dev(gpdev, 0);
+ if (npdev)
+ break;
+ }
+
+ if (!npdev)
+ /* It is not an NPU attached device, skip */
+ return NULL;
>>>
>>> This breaks some configs with:
>>>
>>> arch/powerpc/platforms/powernv/npu-dma.c:550:5: error: 'npdev' may be
>>> used uninitialized in this function [-Werror=uninitialized]
>>
>>
>> gcc 5, 7 and 8 do not warn about this, I have to disable
>> list_for_each_entry() above to recreate this.
>>
>> I even compiled gcc 5.5 which some of your buildmachines use and yet no
>> error on this:
>>
>> make O=/home/aik/pbuild/kernel-le/ KCFLAGS=-Werror=all ARCH=powerpc
>> CROSS_COMPILE=/opt/cross/gcc-powerpc64le-linux-5.5.0-nolibc/bin/powerpc64le-linux-
>> arch/powerpc/platforms/powernv/npu-dma.o
>
> Odd. That error is from kisskb like the others.
>
> http://kisskb.ellerman.id.au/kisskb/buildresult/13622793/
>
> Seems it's GCC 4.6.3 that is producing that one. Not sure why newer
> compilers aren't warning about it.
>
>
> It's pretty obviously correct though, unless you can prove that the list
> is never empty?
I know these are correct and I want to catch these before I post patches :-/
Can I get that gcc 4.6.3? It does not compile on my ubuntu for whatever
reason.
>
> struct pci_dev *npdev;
>
> list_for_each_entry(gpdev, >pbus->devices, bus_list) {
> npdev = pnv_pci_get_npu_dev(gpdev, 0);
> if (npdev)
> break;
> }
>
> if (!npdev)
> /* It is not an NPU attached device, skip */
> return NULL;
>
>
> cheers
>
--
Alexey
Re: [PATCH kernel v5 14/20] powerpc/powernv/npu: Add compound IOMMU groups
Alexey Kardashevskiy writes:
> On 19/12/2018 11:17, Michael Ellerman wrote:
>> Alexey Kardashevskiy writes:
>>> diff --git a/arch/powerpc/platforms/powernv/npu-dma.c
>>> b/arch/powerpc/platforms/powernv/npu-dma.c
>>> index dc629ee..3468eaa 100644
>>> --- a/arch/powerpc/platforms/powernv/npu-dma.c
>>> +++ b/arch/powerpc/platforms/powernv/npu-dma.c
>>> @@ -372,8 +358,263 @@ struct npu {
>> ...
>>> +
>>> +static void pnv_comp_attach_table_group(struct npu_comp *npucomp,
>>> + struct pnv_ioda_pe *pe)
>>> +{
>>> + if (WARN_ON(npucomp->pe_num == NV_NPU_MAX_PE_NUM))
>>> + return;
>>> +
>>> + npucomp->pe[npucomp->pe_num] = pe;
>>> + ++npucomp->pe_num;
>>> +}
>>> +
>>> +struct iommu_table_group *pnv_try_setup_npu_table_group(struct pnv_ioda_pe
>>> *pe)
>>> +{
>>> + struct iommu_table_group *table_group;
>>> + struct npu_comp *npucomp;
>>> + struct pci_dev *gpdev = NULL;
>>> + struct pci_controller *hose;
>>> + struct pci_dev *npdev;
>>> +
>>> + list_for_each_entry(gpdev, >pbus->devices, bus_list) {
>>> + npdev = pnv_pci_get_npu_dev(gpdev, 0);
>>> + if (npdev)
>>> + break;
>>> + }
>>> +
>>> + if (!npdev)
>>> + /* It is not an NPU attached device, skip */
>>> + return NULL;
>>
>> This breaks some configs with:
>>
>> arch/powerpc/platforms/powernv/npu-dma.c:550:5: error: 'npdev' may be used
>> uninitialized in this function [-Werror=uninitialized]
>
>
> gcc 5, 7 and 8 do not warn about this, I have to disable
> list_for_each_entry() above to recreate this.
>
> I even compiled gcc 5.5 which some of your buildmachines use and yet no
> error on this:
>
> make O=/home/aik/pbuild/kernel-le/ KCFLAGS=-Werror=all ARCH=powerpc
> CROSS_COMPILE=/opt/cross/gcc-powerpc64le-linux-5.5.0-nolibc/bin/powerpc64le-linux-
> arch/powerpc/platforms/powernv/npu-dma.o
Odd. That error is from kisskb like the others.
http://kisskb.ellerman.id.au/kisskb/buildresult/13622793/
Seems it's GCC 4.6.3 that is producing that one. Not sure why newer
compilers aren't warning about it.
It's pretty obviously correct though, unless you can prove that the list
is never empty?
struct pci_dev *npdev;
list_for_each_entry(gpdev, >pbus->devices, bus_list) {
npdev = pnv_pci_get_npu_dev(gpdev, 0);
if (npdev)
break;
}
if (!npdev)
/* It is not an NPU attached device, skip */
return NULL;
cheers
Re: [PATCH kernel v5 14/20] powerpc/powernv/npu: Add compound IOMMU groups
On 19/12/2018 11:17, Michael Ellerman wrote:
> Alexey Kardashevskiy writes:
>> diff --git a/arch/powerpc/platforms/powernv/npu-dma.c
>> b/arch/powerpc/platforms/powernv/npu-dma.c
>> index dc629ee..3468eaa 100644
>> --- a/arch/powerpc/platforms/powernv/npu-dma.c
>> +++ b/arch/powerpc/platforms/powernv/npu-dma.c
>> @@ -372,8 +358,263 @@ struct npu {
> ...
>> +
>> +static void pnv_comp_attach_table_group(struct npu_comp *npucomp,
>> +struct pnv_ioda_pe *pe)
>> +{
>> +if (WARN_ON(npucomp->pe_num == NV_NPU_MAX_PE_NUM))
>> +return;
>> +
>> +npucomp->pe[npucomp->pe_num] = pe;
>> +++npucomp->pe_num;
>> +}
>> +
>> +struct iommu_table_group *pnv_try_setup_npu_table_group(struct pnv_ioda_pe
>> *pe)
>> +{
>> +struct iommu_table_group *table_group;
>> +struct npu_comp *npucomp;
>> +struct pci_dev *gpdev = NULL;
>> +struct pci_controller *hose;
>> +struct pci_dev *npdev;
>> +
>> +list_for_each_entry(gpdev, >pbus->devices, bus_list) {
>> +npdev = pnv_pci_get_npu_dev(gpdev, 0);
>> +if (npdev)
>> +break;
>> +}
>> +
>> +if (!npdev)
>> +/* It is not an NPU attached device, skip */
>> +return NULL;
>
> This breaks some configs with:
>
> arch/powerpc/platforms/powernv/npu-dma.c:550:5: error: 'npdev' may be used
> uninitialized in this function [-Werror=uninitialized]
gcc 5, 7 and 8 do not warn about this, I have to disable
list_for_each_entry() above to recreate this.
I even compiled gcc 5.5 which some of your buildmachines use and yet no
error on this:
make O=/home/aik/pbuild/kernel-le/ KCFLAGS=-Werror=all ARCH=powerpc
CROSS_COMPILE=/opt/cross/gcc-powerpc64le-linux-5.5.0-nolibc/bin/powerpc64le-linux-
arch/powerpc/platforms/powernv/npu-dma.o
I only get an error when I do:
@@ -525,6 +525,7 @@ struct iommu_table_group
*pnv_try_setup_npu_table_group(struct pnv_ioda_pe *pe)
struct pci_controller *hose;
struct pci_dev *npdev;
+ if (0)
list_for_each_entry(gpdev, >pbus->devices, bus_list) {
npdev = pnv_pci_get_npu_dev(gpdev, 0);
if (npdev)
How do you compile?
--
Alexey
Re: [PATCH kernel v5 14/20] powerpc/powernv/npu: Add compound IOMMU groups
Alexey Kardashevskiy writes:
> diff --git a/arch/powerpc/platforms/powernv/npu-dma.c
> b/arch/powerpc/platforms/powernv/npu-dma.c
> index dc629ee..3468eaa 100644
> --- a/arch/powerpc/platforms/powernv/npu-dma.c
> +++ b/arch/powerpc/platforms/powernv/npu-dma.c
> @@ -372,8 +358,263 @@ struct npu {
...
> +
> +static void pnv_comp_attach_table_group(struct npu_comp *npucomp,
> + struct pnv_ioda_pe *pe)
> +{
> + if (WARN_ON(npucomp->pe_num == NV_NPU_MAX_PE_NUM))
> + return;
> +
> + npucomp->pe[npucomp->pe_num] = pe;
> + ++npucomp->pe_num;
> +}
> +
> +struct iommu_table_group *pnv_try_setup_npu_table_group(struct pnv_ioda_pe
> *pe)
> +{
> + struct iommu_table_group *table_group;
> + struct npu_comp *npucomp;
> + struct pci_dev *gpdev = NULL;
> + struct pci_controller *hose;
> + struct pci_dev *npdev;
> +
> + list_for_each_entry(gpdev, >pbus->devices, bus_list) {
> + npdev = pnv_pci_get_npu_dev(gpdev, 0);
> + if (npdev)
> + break;
> + }
> +
> + if (!npdev)
> + /* It is not an NPU attached device, skip */
> + return NULL;
This breaks some configs with:
arch/powerpc/platforms/powernv/npu-dma.c:550:5: error: 'npdev' may be used
uninitialized in this function [-Werror=uninitialized]
cheers
[PATCH kernel v5 14/20] powerpc/powernv/npu: Add compound IOMMU groups
At the moment the powernv platform registers an IOMMU group for each PE.
There is an exception though: an NVLink bridge which is attached to
the corresponding GPU's IOMMU group making it a master.
Now we have POWER9 systems with GPUs connected to each other directly
bypassing PCI. At the moment we do not control state of these links so
we have to put such interconnected GPUs to one IOMMU group which
means that the old scheme with one GPU as a master won't work - there will
be up to 3 GPUs in such group.
This introduces a npu_comp struct which represents a compound IOMMU
group made of multiple PEs - PCI PEs (for GPUs) and NPU PEs (for NVLink
bridges). This converts the existing NVLink1 code to use the new scheme.
>From now on, each PE must have a valid iommu_table_group_ops which will
either be called directly (for a single PE group) or indirectly from
a compound group handlers.
This moves IOMMU group registration for NVLink-connected GPUs to npu-dma.c.
For POWER8, this stores a new compound group pointer in the PE (so a GPU
is still a master); for POWER9 the new group pointer is stored in an NPU
(which is allocated per a PCI host controller).
Signed-off-by: Alexey Kardashevskiy
---
Changes:
v5:
* now read page sizes from PHB NVLink to narrow down what the compoind PE
can actually support (hint: 4K/64K only)
---
arch/powerpc/include/asm/pci.h| 1 +
arch/powerpc/platforms/powernv/pci.h | 7 +
arch/powerpc/platforms/powernv/npu-dma.c | 291 --
arch/powerpc/platforms/powernv/pci-ioda.c | 163
4 files changed, 325 insertions(+), 137 deletions(-)
diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h
index baf2886..0c72f18 100644
--- a/arch/powerpc/include/asm/pci.h
+++ b/arch/powerpc/include/asm/pci.h
@@ -132,5 +132,6 @@ extern struct pci_dev *pnv_pci_get_npu_dev(struct pci_dev
*gpdev, int index);
extern int pnv_npu2_init(struct pci_controller *hose);
extern int pnv_npu2_map_lpar_dev(struct pci_dev *gpdev, unsigned int lparid,
unsigned long msr);
+extern int pnv_npu2_unmap_lpar_dev(struct pci_dev *gpdev);
#endif /* __ASM_POWERPC_PCI_H */
diff --git a/arch/powerpc/platforms/powernv/pci.h
b/arch/powerpc/platforms/powernv/pci.h
index cf9f748..aef4bb5 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -62,6 +62,7 @@ struct pnv_ioda_pe {
/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
struct iommu_table_group table_group;
+ struct npu_comp *npucomp;
/* 64-bit TCE bypass region */
booltce_bypass_enabled;
@@ -201,6 +202,8 @@ extern void pnv_teardown_msi_irqs(struct pci_dev *pdev);
extern struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev);
extern void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq);
extern void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
+extern unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
+ __u64 window_size, __u32 levels);
extern int pnv_eeh_post_init(void);
extern void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
@@ -216,6 +219,10 @@ extern void pe_level_printk(const struct pnv_ioda_pe *pe,
const char *level,
extern void pnv_npu_try_dma_set_bypass(struct pci_dev *gpdev, bool bypass);
extern void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_phb *phb, bool rm);
extern struct pnv_ioda_pe *pnv_pci_npu_setup_iommu(struct pnv_ioda_pe *npe);
+extern struct iommu_table_group *pnv_try_setup_npu_table_group(
+ struct pnv_ioda_pe *pe);
+extern struct iommu_table_group *pnv_npu_compound_attach(
+ struct pnv_ioda_pe *pe);
/* pci-ioda-tce.c */
#define POWERNV_IOMMU_DEFAULT_LEVELS 1
diff --git a/arch/powerpc/platforms/powernv/npu-dma.c
b/arch/powerpc/platforms/powernv/npu-dma.c
index dc629ee..3468eaa 100644
--- a/arch/powerpc/platforms/powernv/npu-dma.c
+++ b/arch/powerpc/platforms/powernv/npu-dma.c
@@ -328,31 +328,6 @@ static struct iommu_table_group_ops pnv_pci_npu_ops = {
.unset_window = pnv_npu_unset_window,
.take_ownership = pnv_npu_take_ownership,
};
-
-struct pnv_ioda_pe *pnv_pci_npu_setup_iommu(struct pnv_ioda_pe *npe)
-{
- struct pnv_phb *phb = npe->phb;
- struct pci_bus *pbus = phb->hose->bus;
- struct pci_dev *npdev, *gpdev = NULL, *gptmp;
- struct pnv_ioda_pe *gpe = get_gpu_pci_dev_and_pe(npe, );
-
- if (!gpe || !gpdev)
- return NULL;
-
- npe->table_group.ops = _pci_npu_ops;
-
- list_for_each_entry(npdev, >devices, bus_list) {
- gptmp = pnv_pci_get_gpu_dev(npdev);
-
- if (gptmp != gpdev)
- continue;
-
- pe_info(gpe, "Attached NPU %s\n", dev_name(>dev));
- iommu_group_add_device(gpe->table_group.group, >dev);
- }
-
- return gpe;
-}
#endif /* !CONFIG_IOMMU_API */
/*
@@ -360,6 +335,17 @@
