Re: [PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
On Mon, 2020-03-02 at 18:51 +0100, Frederic Barrat wrote: > > Le 21/02/2020 à 04:27, Alastair D'Silva a écrit : > > From: Alastair D'Silva > > > > This patch reads timeouts & firmware version from the controller, > > and > > uses those timeouts to wait for the controller to report that it is > > ready > > before handing the memory over to libnvdimm. > > > > Signed-off-by: Alastair D'Silva > > --- > > arch/powerpc/platforms/powernv/pmem/Makefile | 2 +- > > arch/powerpc/platforms/powernv/pmem/ocxl.c| 92 > > +++ > > .../platforms/powernv/pmem/ocxl_internal.c| 19 > > .../platforms/powernv/pmem/ocxl_internal.h| 24 + > > 4 files changed, 136 insertions(+), 1 deletion(-) > > create mode 100644 > > arch/powerpc/platforms/powernv/pmem/ocxl_internal.c > > > > diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile > > b/arch/powerpc/platforms/powernv/pmem/Makefile > > index 1c55c4193175..4ceda25907d4 100644 > > --- a/arch/powerpc/platforms/powernv/pmem/Makefile > > +++ b/arch/powerpc/platforms/powernv/pmem/Makefile > > @@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror > > > > obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o > > > > -ocxlpmem-y := ocxl.o > > +ocxlpmem-y := ocxl.o ocxl_internal.o > > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c > > b/arch/powerpc/platforms/powernv/pmem/ocxl.c > > index 3c4eeb5dcc0f..431212c9f0cc 100644 > > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c > > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c > > @@ -8,6 +8,7 @@ > > > > #include > > #include > > +#include > > #include > > #include > > #include > > @@ -215,6 +216,36 @@ static int register_lpc_mem(struct ocxlpmem > > *ocxlpmem) > > return 0; > > } > > > > +/** > > + * is_usable() - Is a controller usable? > > + * @ocxlpmem: the device metadata > > + * @verbose: True to log errors > > + * Return: true if the controller is usable > > + */ > > +static bool is_usable(const struct ocxlpmem *ocxlpmem, bool > > verbose) > > +{ > > + u64 chi = 0; > > + int rc = ocxlpmem_chi(ocxlpmem, ); > > + > > + if (rc < 0) > > + return false; > > + > > + if (!(chi & GLOBAL_MMIO_CHI_CRDY)) { > > + if (verbose) > > + dev_err(>dev, "controller is not > > ready.\n"); > > + return false; > > + } > > + > > + if (!(chi & GLOBAL_MMIO_CHI_MA)) { > > + if (verbose) > > + dev_err(>dev, > > + "controller does not have memory > > available.\n"); > > + return false; > > + } > > + > > + return true; > > +} > > + > > /** > >* allocate_minor() - Allocate a minor number to use for an > > OpenCAPI pmem device > >* @ocxlpmem: the device metadata > > @@ -328,6 +359,48 @@ static void ocxlpmem_remove(struct pci_dev > > *pdev) > > } > > } > > > > +/** > > + * read_device_metadata() - Retrieve config information from the > > AFU and save it for future use > > + * @ocxlpmem: the device metadata > > + * Return: 0 on success, negative on failure > > + */ > > +static int read_device_metadata(struct ocxlpmem *ocxlpmem) > > +{ > > + u64 val; > > + int rc; > > + > > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, > > GLOBAL_MMIO_CCAP0, > > +OCXL_LITTLE_ENDIAN, ); > > + if (rc) > > + return rc; > > + > > + ocxlpmem->scm_revision = val & 0x; > > + ocxlpmem->read_latency = (val >> 32) & 0xFF; > > + ocxlpmem->readiness_timeout = (val >> 48) & 0x0F; > > + ocxlpmem->memory_available_timeout = val >> 52; > > + > > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, > > GLOBAL_MMIO_CCAP1, > > +OCXL_LITTLE_ENDIAN, ); > > + if (rc) > > + return rc; > > + > > + ocxlpmem->max_controller_dump_size = val & 0x; > > + > > + // Extract firmware version text > > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, > > GLOBAL_MMIO_FWVER, > > +OCXL_HOST_ENDIAN, (u64 *)ocxlpmem- > > >fw_version); > > + if (rc) > > + return rc; > > + > > + ocxlpmem->fw_version[8] = '\0'; > > + > > + dev_info(>dev, > > +"Firmware version '%s' SCM revision %d:%d\n", > > ocxlpmem->fw_version, > > +ocxlpmem->scm_revision >> 4, ocxlpmem->scm_revision & > > 0x0F); > > + > > + return 0; > > +} > > + > > /** > >* probe_function0() - Set up function 0 for an OpenCAPI > > persistent memory device > >* This is important as it enables templates higher than 0 across > > all other functions, > > @@ -368,6 +441,7 @@ static int probe(struct pci_dev *pdev, const > > struct pci_device_id *ent) > > { > > struct ocxlpmem *ocxlpmem; > > int rc; > > + u16 elapsed, timeout; > > > > if (PCI_FUNC(pdev->devfn) == 0) > > return probe_function0(pdev); > > @@ -422,6 +496,24 @@ static int probe(struct pci_dev *pdev, const > > struct pci_device_id *ent) > >
Re: [PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
Le 21/02/2020 à 04:27, Alastair D'Silva a écrit : From: Alastair D'Silva This patch reads timeouts & firmware version from the controller, and uses those timeouts to wait for the controller to report that it is ready before handing the memory over to libnvdimm. Signed-off-by: Alastair D'Silva --- arch/powerpc/platforms/powernv/pmem/Makefile | 2 +- arch/powerpc/platforms/powernv/pmem/ocxl.c| 92 +++ .../platforms/powernv/pmem/ocxl_internal.c| 19 .../platforms/powernv/pmem/ocxl_internal.h| 24 + 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_internal.c diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile b/arch/powerpc/platforms/powernv/pmem/Makefile index 1c55c4193175..4ceda25907d4 100644 --- a/arch/powerpc/platforms/powernv/pmem/Makefile +++ b/arch/powerpc/platforms/powernv/pmem/Makefile @@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o -ocxlpmem-y := ocxl.o +ocxlpmem-y := ocxl.o ocxl_internal.o diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c b/arch/powerpc/platforms/powernv/pmem/ocxl.c index 3c4eeb5dcc0f..431212c9f0cc 100644 --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -215,6 +216,36 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem) return 0; } +/** + * is_usable() - Is a controller usable? + * @ocxlpmem: the device metadata + * @verbose: True to log errors + * Return: true if the controller is usable + */ +static bool is_usable(const struct ocxlpmem *ocxlpmem, bool verbose) +{ + u64 chi = 0; + int rc = ocxlpmem_chi(ocxlpmem, ); + + if (rc < 0) + return false; + + if (!(chi & GLOBAL_MMIO_CHI_CRDY)) { + if (verbose) + dev_err(>dev, "controller is not ready.\n"); + return false; + } + + if (!(chi & GLOBAL_MMIO_CHI_MA)) { + if (verbose) + dev_err(>dev, + "controller does not have memory available.\n"); + return false; + } + + return true; +} + /** * allocate_minor() - Allocate a minor number to use for an OpenCAPI pmem device * @ocxlpmem: the device metadata @@ -328,6 +359,48 @@ static void ocxlpmem_remove(struct pci_dev *pdev) } } +/** + * read_device_metadata() - Retrieve config information from the AFU and save it for future use + * @ocxlpmem: the device metadata + * Return: 0 on success, negative on failure + */ +static int read_device_metadata(struct ocxlpmem *ocxlpmem) +{ + u64 val; + int rc; + + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP0, +OCXL_LITTLE_ENDIAN, ); + if (rc) + return rc; + + ocxlpmem->scm_revision = val & 0x; + ocxlpmem->read_latency = (val >> 32) & 0xFF; + ocxlpmem->readiness_timeout = (val >> 48) & 0x0F; + ocxlpmem->memory_available_timeout = val >> 52; + + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP1, +OCXL_LITTLE_ENDIAN, ); + if (rc) + return rc; + + ocxlpmem->max_controller_dump_size = val & 0x; + + // Extract firmware version text + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_FWVER, +OCXL_HOST_ENDIAN, (u64 *)ocxlpmem->fw_version); + if (rc) + return rc; + + ocxlpmem->fw_version[8] = '\0'; + + dev_info(>dev, +"Firmware version '%s' SCM revision %d:%d\n", ocxlpmem->fw_version, +ocxlpmem->scm_revision >> 4, ocxlpmem->scm_revision & 0x0F); + + return 0; +} + /** * probe_function0() - Set up function 0 for an OpenCAPI persistent memory device * This is important as it enables templates higher than 0 across all other functions, @@ -368,6 +441,7 @@ static int probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct ocxlpmem *ocxlpmem; int rc; + u16 elapsed, timeout; if (PCI_FUNC(pdev->devfn) == 0) return probe_function0(pdev); @@ -422,6 +496,24 @@ static int probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err; } + if (read_device_metadata(ocxlpmem)) { + dev_err(>dev, "Could not read metadata\n"); Need to set rc + goto err; + } + + elapsed = 0; + timeout = ocxlpmem->readiness_timeout + ocxlpmem->memory_available_timeout; + while (!is_usable(ocxlpmem, false)) { + if (elapsed++ > timeout) { + dev_warn(>dev, "OpenCAPI Persistent Memory ready timeout.\n"); +
Re: [PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
On Thu, 2020-02-27 at 14:54 +1100, Andrew Donnellan wrote: > On 21/2/20 2:27 pm, Alastair D'Silva wrote: > > +/** > > + * read_device_metadata() - Retrieve config information from the > > AFU and save it for future use > > + * @ocxlpmem: the device metadata > > + * Return: 0 on success, negative on failure > > + */ > > +static int read_device_metadata(struct ocxlpmem *ocxlpmem) > > +{ > > + u64 val; > > + int rc; > > + > > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, > > GLOBAL_MMIO_CCAP0, > > +OCXL_LITTLE_ENDIAN, ); > > + if (rc) > > + return rc; > > + > > + ocxlpmem->scm_revision = val & 0x; > > + ocxlpmem->read_latency = (val >> 32) & 0xFF; > > This field is 16 bits in the spec, so the mask should be 0x I > think? > You're right, I'll fix it. > Maybe we should generalise the EXTRACT_BITS macro we use in ocxl :) > -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819
Re: [PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
On 21/2/20 2:27 pm, Alastair D'Silva wrote: +/** + * read_device_metadata() - Retrieve config information from the AFU and save it for future use + * @ocxlpmem: the device metadata + * Return: 0 on success, negative on failure + */ +static int read_device_metadata(struct ocxlpmem *ocxlpmem) +{ + u64 val; + int rc; + + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP0, +OCXL_LITTLE_ENDIAN, ); + if (rc) + return rc; + + ocxlpmem->scm_revision = val & 0x; + ocxlpmem->read_latency = (val >> 32) & 0xFF; This field is 16 bits in the spec, so the mask should be 0x I think? Maybe we should generalise the EXTRACT_BITS macro we use in ocxl :) -- Andrew Donnellan OzLabs, ADL Canberra a...@linux.ibm.com IBM Australia Limited
[PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
From: Alastair D'Silva This patch reads timeouts & firmware version from the controller, and uses those timeouts to wait for the controller to report that it is ready before handing the memory over to libnvdimm. Signed-off-by: Alastair D'Silva --- arch/powerpc/platforms/powernv/pmem/Makefile | 2 +- arch/powerpc/platforms/powernv/pmem/ocxl.c| 92 +++ .../platforms/powernv/pmem/ocxl_internal.c| 19 .../platforms/powernv/pmem/ocxl_internal.h| 24 + 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_internal.c diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile b/arch/powerpc/platforms/powernv/pmem/Makefile index 1c55c4193175..4ceda25907d4 100644 --- a/arch/powerpc/platforms/powernv/pmem/Makefile +++ b/arch/powerpc/platforms/powernv/pmem/Makefile @@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o -ocxlpmem-y := ocxl.o +ocxlpmem-y := ocxl.o ocxl_internal.o diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c b/arch/powerpc/platforms/powernv/pmem/ocxl.c index 3c4eeb5dcc0f..431212c9f0cc 100644 --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -215,6 +216,36 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem) return 0; } +/** + * is_usable() - Is a controller usable? + * @ocxlpmem: the device metadata + * @verbose: True to log errors + * Return: true if the controller is usable + */ +static bool is_usable(const struct ocxlpmem *ocxlpmem, bool verbose) +{ + u64 chi = 0; + int rc = ocxlpmem_chi(ocxlpmem, ); + + if (rc < 0) + return false; + + if (!(chi & GLOBAL_MMIO_CHI_CRDY)) { + if (verbose) + dev_err(>dev, "controller is not ready.\n"); + return false; + } + + if (!(chi & GLOBAL_MMIO_CHI_MA)) { + if (verbose) + dev_err(>dev, + "controller does not have memory available.\n"); + return false; + } + + return true; +} + /** * allocate_minor() - Allocate a minor number to use for an OpenCAPI pmem device * @ocxlpmem: the device metadata @@ -328,6 +359,48 @@ static void ocxlpmem_remove(struct pci_dev *pdev) } } +/** + * read_device_metadata() - Retrieve config information from the AFU and save it for future use + * @ocxlpmem: the device metadata + * Return: 0 on success, negative on failure + */ +static int read_device_metadata(struct ocxlpmem *ocxlpmem) +{ + u64 val; + int rc; + + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP0, +OCXL_LITTLE_ENDIAN, ); + if (rc) + return rc; + + ocxlpmem->scm_revision = val & 0x; + ocxlpmem->read_latency = (val >> 32) & 0xFF; + ocxlpmem->readiness_timeout = (val >> 48) & 0x0F; + ocxlpmem->memory_available_timeout = val >> 52; + + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP1, +OCXL_LITTLE_ENDIAN, ); + if (rc) + return rc; + + ocxlpmem->max_controller_dump_size = val & 0x; + + // Extract firmware version text + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_FWVER, +OCXL_HOST_ENDIAN, (u64 *)ocxlpmem->fw_version); + if (rc) + return rc; + + ocxlpmem->fw_version[8] = '\0'; + + dev_info(>dev, +"Firmware version '%s' SCM revision %d:%d\n", ocxlpmem->fw_version, +ocxlpmem->scm_revision >> 4, ocxlpmem->scm_revision & 0x0F); + + return 0; +} + /** * probe_function0() - Set up function 0 for an OpenCAPI persistent memory device * This is important as it enables templates higher than 0 across all other functions, @@ -368,6 +441,7 @@ static int probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct ocxlpmem *ocxlpmem; int rc; + u16 elapsed, timeout; if (PCI_FUNC(pdev->devfn) == 0) return probe_function0(pdev); @@ -422,6 +496,24 @@ static int probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err; } + if (read_device_metadata(ocxlpmem)) { + dev_err(>dev, "Could not read metadata\n"); + goto err; + } + + elapsed = 0; + timeout = ocxlpmem->readiness_timeout + ocxlpmem->memory_available_timeout; + while (!is_usable(ocxlpmem, false)) { + if (elapsed++ > timeout) { + dev_warn(>dev, "OpenCAPI Persistent Memory ready timeout.\n"); + (void)is_usable(ocxlpmem, true); + rc = -ENXIO; +