[PATCH 3/3] papr/scm: Add bad memory ranges to nvdimm bad ranges

2019-08-19 Thread Santosh Sivaraj
Subscribe to the MCE notification and add the physical address which
generated a memory error to nvdimm bad range.

Signed-off-by: Santosh Sivaraj 
---
 arch/powerpc/platforms/pseries/papr_scm.c | 86 ++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/platforms/pseries/papr_scm.c 
b/arch/powerpc/platforms/pseries/papr_scm.c
index a5ac371a3f06..e38f7febc5d9 100644
--- a/arch/powerpc/platforms/pseries/papr_scm.c
+++ b/arch/powerpc/platforms/pseries/papr_scm.c
@@ -12,6 +12,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #include 
 
@@ -39,8 +41,12 @@ struct papr_scm_priv {
struct resource res;
struct nd_region *region;
struct nd_interleave_set nd_set;
+   struct list_head region_list;
 };
 
+LIST_HEAD(papr_nd_regions);
+DEFINE_MUTEX(papr_ndr_lock);
+
 static int drc_pmem_bind(struct papr_scm_priv *p)
 {
unsigned long ret[PLPAR_HCALL_BUFSIZE];
@@ -364,6 +370,10 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
dev_info(dev, "Region registered with target node %d and online 
node %d",
 target_nid, online_nid);
 
+   mutex_lock(_ndr_lock);
+   list_add_tail(>region_list, _nd_regions);
+   mutex_unlock(_ndr_lock);
+
return 0;
 
 err:   nvdimm_bus_unregister(p->bus);
@@ -371,6 +381,57 @@ err:   nvdimm_bus_unregister(p->bus);
return -ENXIO;
 }
 
+static int handle_mce_ue(struct notifier_block *nb, unsigned long val,
+void *data)
+{
+   struct machine_check_event *evt = data;
+   struct papr_scm_priv *p;
+   u64 phys_addr, aligned_addr;
+   bool found = false;
+
+   if (evt->error_type != MCE_ERROR_TYPE_UE)
+   return NOTIFY_DONE;
+
+   if (list_empty(_nd_regions))
+   return NOTIFY_DONE;
+
+   phys_addr = evt->u.ue_error.physical_address +
+   (evt->u.ue_error.effective_address & ~PAGE_MASK);
+
+   if (!evt->u.ue_error.physical_address_provided ||
+   !is_zone_device_page(pfn_to_page(phys_addr >> PAGE_SHIFT)))
+   return NOTIFY_DONE;
+
+   mutex_lock(_ndr_lock);
+   list_for_each_entry(p, _nd_regions, region_list) {
+   struct resource res = p->res;
+
+   if (phys_addr >= res.start && phys_addr <= res.end) {
+   found = true;
+   break;
+   }
+   }
+   mutex_unlock(_ndr_lock);
+
+   if (!found)
+   return NOTIFY_DONE;
+
+   aligned_addr = ALIGN_DOWN(phys_addr, L1_CACHE_BYTES);
+   if (nvdimm_bus_add_badrange(p->bus, aligned_addr, L1_CACHE_BYTES))
+   return NOTIFY_DONE;
+
+   pr_debug("Add memory range (0x%llx -- 0x%llx) as bad range\n",
+aligned_addr, aligned_addr + L1_CACHE_BYTES);
+
+   nvdimm_region_notify(p->region, NVDIMM_REVALIDATE_POISON);
+
+   return NOTIFY_OK;
+}
+
+static struct notifier_block mce_ue_nb = {
+   .notifier_call = handle_mce_ue
+};
+
 static int papr_scm_probe(struct platform_device *pdev)
 {
struct device_node *dn = pdev->dev.of_node;
@@ -456,6 +517,7 @@ static int papr_scm_probe(struct platform_device *pdev)
goto err2;
 
platform_set_drvdata(pdev, p);
+   mce_register_notifier(_ue_nb);
 
return 0;
 
@@ -468,6 +530,10 @@ static int papr_scm_remove(struct platform_device *pdev)
 {
struct papr_scm_priv *p = platform_get_drvdata(pdev);
 
+   mutex_lock(_ndr_lock);
+   list_del(&(p->region_list));
+   mutex_unlock(_ndr_lock);
+
nvdimm_bus_unregister(p->bus);
drc_pmem_unbind(p);
kfree(p);
@@ -490,7 +556,25 @@ static struct platform_driver papr_scm_driver = {
},
 };
 
-module_platform_driver(papr_scm_driver);
+static int __init papr_scm_init(void)
+{
+   int ret;
+
+   ret = platform_driver_register(_scm_driver);
+   if (!ret)
+   mce_register_notifier(_ue_nb);
+
+   return ret;
+}
+module_init(papr_scm_init);
+
+static void __exit papr_scm_exit(void)
+{
+   mce_unregister_notifier(_ue_nb);
+   platform_driver_unregister(_scm_driver);
+}
+module_exit(papr_scm_exit);
+
 MODULE_DEVICE_TABLE(of, papr_scm_match);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("IBM Corporation");
-- 
2.21.0



Re: [PATCH 3/3] papr/scm: Add bad memory ranges to nvdimm bad ranges

2019-08-15 Thread Santosh Sivaraj
"Oliver O'Halloran"  writes:

> On Wed, Aug 14, 2019 at 6:25 PM Santosh Sivaraj  wrote:
>>
>> Subscribe to the MCE notification and add the physical address which
>> generated a memory error to nvdimm bad range.
>>
>> Signed-off-by: Santosh Sivaraj 
>> ---
>>  arch/powerpc/platforms/pseries/papr_scm.c | 65 +++
>>  1 file changed, 65 insertions(+)
>>
>> diff --git a/arch/powerpc/platforms/pseries/papr_scm.c 
>> b/arch/powerpc/platforms/pseries/papr_scm.c
>> index a5ac371a3f06..4d25c98a9835 100644
>> --- a/arch/powerpc/platforms/pseries/papr_scm.c
>> +++ b/arch/powerpc/platforms/pseries/papr_scm.c
>> @@ -12,6 +12,8 @@
>>  #include 
>>  #include 
>>  #include 
>> +#include 
>> +#include 
>>
>>  #include 
>>
>> @@ -39,8 +41,12 @@ struct papr_scm_priv {
>> struct resource res;
>> struct nd_region *region;
>> struct nd_interleave_set nd_set;
>> +   struct list_head list;
>
> list is not a meaningful name. call it something more descriptive.
>
>>  };
>>
>> +LIST_HEAD(papr_nd_regions);
>> +DEFINE_MUTEX(papr_ndr_lock);
>
> Should this be a mutex or a spinlock? I don't know what context the
> mce notifier is called from, but if it's not sleepable then a mutex
> will cause problems. Did you test this with lockdep enabled?

This would be a mutex, we are called from a blocking notifier.

>
>> +
>>  static int drc_pmem_bind(struct papr_scm_priv *p)
>>  {
>> unsigned long ret[PLPAR_HCALL_BUFSIZE];
>> @@ -364,6 +370,10 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
>> dev_info(dev, "Region registered with target node %d and 
>> online node %d",
>>  target_nid, online_nid);
>>
>> +   mutex_lock(_ndr_lock);
>> +   list_add_tail(>list, _nd_regions);
>> +   mutex_unlock(_ndr_lock);
>> +
>
> Where's the matching remove when we unbind the driver?

Missed it completely. Will fix it.

>
>> return 0;
>>pp
>>  err:   nvdimm_bus_unregister(p->bus);
>> @@ -371,6 +381,60 @@ err:   nvdimm_bus_unregister(p->bus);
>> return -ENXIO;
>>  }
>>
>> +static int handle_mce_ue(struct notifier_block *nb, unsigned long val,
>> +void *data)
>> +{
>> +   struct machine_check_event *evt = data;
>> +   struct papr_scm_priv *p;
>> +   u64 phys_addr;
>> +
>> +   if (evt->error_type != MCE_ERROR_TYPE_UE)
>> +   return NOTIFY_DONE;
>> +
>> +   if (list_empty(_nd_regions))
>> +   return NOTIFY_DONE;
>> +
>> +   phys_addr = evt->u.ue_error.physical_address +
>> +   (evt->u.ue_error.effective_address & ~PAGE_MASK);
>
> Wait what? Why is physical_address page aligned, but effective_address
> not? Not a problem with this patch, but still, what the hell?

Not sure why, but its the way now. I can see if I can update it if it makes
sense in a later patch.

>
>> +   if (!evt->u.ue_error.physical_address_provided ||
>> +   !is_zone_device_page(pfn_to_page(phys_addr >> PAGE_SHIFT)))
>> +   return NOTIFY_DONE;
>> +
>> +   mutex_lock(_ndr_lock);
>> +   list_for_each_entry(p, _nd_regions, list) {
>> +   struct resource res = p->res;
>> +   u64 aligned_addr;
>> +
>
>> +   if (res.start > phys_addr)
>> +   continue;
>> +
>> +   if (res.end < phys_addr)
>> +   continue;
>
> surely there's a helper for this
>
>> +
>> +   aligned_addr = ALIGN_DOWN(phys_addr, L1_CACHE_BYTES);
>> +   pr_debug("Add memory range (0x%llx -- 0x%llx) as bad 
>> range\n",
>> +aligned_addr, aligned_addr + L1_CACHE_BYTES);
>> +
>> +   if (nvdimm_bus_add_badrange(p->bus,
>> +   aligned_addr, L1_CACHE_BYTES))
>> +   pr_warn("Failed to add bad range (0x%llx -- 
>> 0x%llx)\n",
>> +   aligned_addr, aligned_addr + L1_CACHE_BYTES);
>> +
>> +   nvdimm_region_notify(p->region,
>> +NVDIMM_REVALIDATE_POISON);
>> +
>> +   break;
>
> nit: you can avoid stacking indetation levels by breaking out of the
> loop as soon as you've found the region you're looking for.

True.

>
>> +   }
>> +   mutex_unlock(_ndr_lock);
>> +
>> +   return NOTIFY_OK;
>> +}
>> +
>> +static struct notifier_block mce_ue_nb = {
>> +   .notifier_call = handle_mce_ue
>> +};
>> +
>>  static int papr_scm_probe(struct platform_device *pdev)
>>  {
>> struct device_node *dn = pdev->dev.of_node;
>> @@ -456,6 +520,7 @@ static int papr_scm_probe(struct platform_device *pdev)
>> goto err2;
>>
>> platform_set_drvdata(pdev, p);
>> +   mce_register_notifier(_ue_nb);
>
> Either get rid of the global region list and have a notifier block in
> each device's driver private data, or keep the global list and
> register the notifier in module_init(). Re-registering the 

Re: [PATCH 3/3] papr/scm: Add bad memory ranges to nvdimm bad ranges

2019-08-15 Thread Oliver O'Halloran
On Wed, Aug 14, 2019 at 6:25 PM Santosh Sivaraj  wrote:
>
> Subscribe to the MCE notification and add the physical address which
> generated a memory error to nvdimm bad range.
>
> Signed-off-by: Santosh Sivaraj 
> ---
>  arch/powerpc/platforms/pseries/papr_scm.c | 65 +++
>  1 file changed, 65 insertions(+)
>
> diff --git a/arch/powerpc/platforms/pseries/papr_scm.c 
> b/arch/powerpc/platforms/pseries/papr_scm.c
> index a5ac371a3f06..4d25c98a9835 100644
> --- a/arch/powerpc/platforms/pseries/papr_scm.c
> +++ b/arch/powerpc/platforms/pseries/papr_scm.c
> @@ -12,6 +12,8 @@
>  #include 
>  #include 
>  #include 
> +#include 
> +#include 
>
>  #include 
>
> @@ -39,8 +41,12 @@ struct papr_scm_priv {
> struct resource res;
> struct nd_region *region;
> struct nd_interleave_set nd_set;
> +   struct list_head list;

list is not a meaningful name. call it something more descriptive.

>  };
>
> +LIST_HEAD(papr_nd_regions);
> +DEFINE_MUTEX(papr_ndr_lock);

Should this be a mutex or a spinlock? I don't know what context the
mce notifier is called from, but if it's not sleepable then a mutex
will cause problems. Did you test this with lockdep enabled?

> +
>  static int drc_pmem_bind(struct papr_scm_priv *p)
>  {
> unsigned long ret[PLPAR_HCALL_BUFSIZE];
> @@ -364,6 +370,10 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
> dev_info(dev, "Region registered with target node %d and 
> online node %d",
>  target_nid, online_nid);
>
> +   mutex_lock(_ndr_lock);
> +   list_add_tail(>list, _nd_regions);
> +   mutex_unlock(_ndr_lock);
> +

Where's the matching remove when we unbind the driver?

> return 0;
>
>  err:   nvdimm_bus_unregister(p->bus);
> @@ -371,6 +381,60 @@ err:   nvdimm_bus_unregister(p->bus);
> return -ENXIO;
>  }
>
> +static int handle_mce_ue(struct notifier_block *nb, unsigned long val,
> +void *data)
> +{
> +   struct machine_check_event *evt = data;
> +   struct papr_scm_priv *p;
> +   u64 phys_addr;
> +
> +   if (evt->error_type != MCE_ERROR_TYPE_UE)
> +   return NOTIFY_DONE;
> +
> +   if (list_empty(_nd_regions))
> +   return NOTIFY_DONE;
> +
> +   phys_addr = evt->u.ue_error.physical_address +
> +   (evt->u.ue_error.effective_address & ~PAGE_MASK);

Wait what? Why is physical_address page aligned, but effective_address
not? Not a problem with this patch, but still, what the hell?

> +   if (!evt->u.ue_error.physical_address_provided ||
> +   !is_zone_device_page(pfn_to_page(phys_addr >> PAGE_SHIFT)))
> +   return NOTIFY_DONE;
> +
> +   mutex_lock(_ndr_lock);
> +   list_for_each_entry(p, _nd_regions, list) {
> +   struct resource res = p->res;
> +   u64 aligned_addr;
> +

> +   if (res.start > phys_addr)
> +   continue;
> +
> +   if (res.end < phys_addr)
> +   continue;

surely there's a helper for this

> +
> +   aligned_addr = ALIGN_DOWN(phys_addr, L1_CACHE_BYTES);
> +   pr_debug("Add memory range (0x%llx -- 0x%llx) as bad range\n",
> +aligned_addr, aligned_addr + L1_CACHE_BYTES);
> +
> +   if (nvdimm_bus_add_badrange(p->bus,
> +   aligned_addr, L1_CACHE_BYTES))
> +   pr_warn("Failed to add bad range (0x%llx -- 
> 0x%llx)\n",
> +   aligned_addr, aligned_addr + L1_CACHE_BYTES);
> +
> +   nvdimm_region_notify(p->region,
> +NVDIMM_REVALIDATE_POISON);
> +
> +   break;

nit: you can avoid stacking indetation levels by breaking out of the
loop as soon as you've found the region you're looking for.

> +   }
> +   mutex_unlock(_ndr_lock);
> +
> +   return NOTIFY_OK;
> +}
> +
> +static struct notifier_block mce_ue_nb = {
> +   .notifier_call = handle_mce_ue
> +};
> +
>  static int papr_scm_probe(struct platform_device *pdev)
>  {
> struct device_node *dn = pdev->dev.of_node;
> @@ -456,6 +520,7 @@ static int papr_scm_probe(struct platform_device *pdev)
> goto err2;
>
> platform_set_drvdata(pdev, p);
> +   mce_register_notifier(_ue_nb);

Either get rid of the global region list and have a notifier block in
each device's driver private data, or keep the global list and
register the notifier in module_init(). Re-registering the notifier
each time a seperate device is probed seems very sketchy.

> return 0;
>
> --
> 2.21.0
>


[PATCH 3/3] papr/scm: Add bad memory ranges to nvdimm bad ranges

2019-08-14 Thread Santosh Sivaraj
Subscribe to the MCE notification and add the physical address which
generated a memory error to nvdimm bad range.

Signed-off-by: Santosh Sivaraj 
---
 arch/powerpc/platforms/pseries/papr_scm.c | 65 +++
 1 file changed, 65 insertions(+)

diff --git a/arch/powerpc/platforms/pseries/papr_scm.c 
b/arch/powerpc/platforms/pseries/papr_scm.c
index a5ac371a3f06..4d25c98a9835 100644
--- a/arch/powerpc/platforms/pseries/papr_scm.c
+++ b/arch/powerpc/platforms/pseries/papr_scm.c
@@ -12,6 +12,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #include 
 
@@ -39,8 +41,12 @@ struct papr_scm_priv {
struct resource res;
struct nd_region *region;
struct nd_interleave_set nd_set;
+   struct list_head list;
 };
 
+LIST_HEAD(papr_nd_regions);
+DEFINE_MUTEX(papr_ndr_lock);
+
 static int drc_pmem_bind(struct papr_scm_priv *p)
 {
unsigned long ret[PLPAR_HCALL_BUFSIZE];
@@ -364,6 +370,10 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
dev_info(dev, "Region registered with target node %d and online 
node %d",
 target_nid, online_nid);
 
+   mutex_lock(_ndr_lock);
+   list_add_tail(>list, _nd_regions);
+   mutex_unlock(_ndr_lock);
+
return 0;
 
 err:   nvdimm_bus_unregister(p->bus);
@@ -371,6 +381,60 @@ err:   nvdimm_bus_unregister(p->bus);
return -ENXIO;
 }
 
+static int handle_mce_ue(struct notifier_block *nb, unsigned long val,
+void *data)
+{
+   struct machine_check_event *evt = data;
+   struct papr_scm_priv *p;
+   u64 phys_addr;
+
+   if (evt->error_type != MCE_ERROR_TYPE_UE)
+   return NOTIFY_DONE;
+
+   if (list_empty(_nd_regions))
+   return NOTIFY_DONE;
+
+   phys_addr = evt->u.ue_error.physical_address +
+   (evt->u.ue_error.effective_address & ~PAGE_MASK);
+
+   if (!evt->u.ue_error.physical_address_provided ||
+   !is_zone_device_page(pfn_to_page(phys_addr >> PAGE_SHIFT)))
+   return NOTIFY_DONE;
+
+   mutex_lock(_ndr_lock);
+   list_for_each_entry(p, _nd_regions, list) {
+   struct resource res = p->res;
+   u64 aligned_addr;
+
+   if (res.start > phys_addr)
+   continue;
+
+   if (res.end < phys_addr)
+   continue;
+
+   aligned_addr = ALIGN_DOWN(phys_addr, L1_CACHE_BYTES);
+   pr_debug("Add memory range (0x%llx -- 0x%llx) as bad range\n",
+aligned_addr, aligned_addr + L1_CACHE_BYTES);
+
+   if (nvdimm_bus_add_badrange(p->bus,
+   aligned_addr, L1_CACHE_BYTES))
+   pr_warn("Failed to add bad range (0x%llx -- 0x%llx)\n",
+   aligned_addr, aligned_addr + L1_CACHE_BYTES);
+
+   nvdimm_region_notify(p->region,
+NVDIMM_REVALIDATE_POISON);
+
+   break;
+   }
+   mutex_unlock(_ndr_lock);
+
+   return NOTIFY_OK;
+}
+
+static struct notifier_block mce_ue_nb = {
+   .notifier_call = handle_mce_ue
+};
+
 static int papr_scm_probe(struct platform_device *pdev)
 {
struct device_node *dn = pdev->dev.of_node;
@@ -456,6 +520,7 @@ static int papr_scm_probe(struct platform_device *pdev)
goto err2;
 
platform_set_drvdata(pdev, p);
+   mce_register_notifier(_ue_nb);
 
return 0;
 
-- 
2.21.0