Re: [PATCHv2] coresight: etm4x: Fix etm4_count race by moving cpuhp callbacks to init
Hi Mike, On 2020-07-29 01:46, Mike Leach wrote: Hi Sai, On Tue, 28 Jul 2020 at 08:51, Sai Prakash Ranjan wrote: etm4_count keeps track of number of ETMv4 registered and on some systems, a race is observed on etm4_count variable which can lead to multiple calls to cpuhp_setup_state_nocalls_cpuslocked(). This function internally calls cpuhp_store_callbacks() which prevents multiple registrations of callbacks for a given state and due to this race, it returns -EBUSY leading to ETM probe failures like below. coresight-etm4x: probe of 704.etm failed with error -16 This race can easily be triggered with async probe by setting probe type as PROBE_PREFER_ASYNCHRONOUS and with ETM power management property "arm,coresight-loses-context-with-cpu". Prevent this race by moving cpuhp callbacks to etm driver init since the cpuhp callbacks doesn't have to depend on the etm4_count and can be once setup during driver init. Similarly we move cpu_pm notifier registration to driver init and completely remove etm4_count usage. Fixes: 9b6a3f3633a5 ("coresight: etmv4: Fix CPU power management setup in probe() function") Fixes: 58eb457be028 ("hwtracing/coresight-etm4x: Convert to hotplug state machine") Suggested-by: Suzuki K Poulose Signed-off-by: Sai Prakash Ranjan --- Changes in v2: * Rearrange cpuhp callbacks and move them to driver init (Suzuki K Poulose) --- drivers/hwtracing/coresight/coresight-etm4x.c | 51 ++- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c index 6d7d2169bfb2..adb71987a1e3 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.c +++ b/drivers/hwtracing/coresight/coresight-etm4x.c @@ -48,8 +48,6 @@ module_param(pm_save_enable, int, 0444); MODULE_PARM_DESC(pm_save_enable, "Save/restore state on power down: 1 = never, 2 = self-hosted"); -/* The number of ETMv4 currently registered */ -static int etm4_count; static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; static void etm4_set_default_config(struct etmv4_config *config); static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, @@ -1403,12 +1401,9 @@ static int etm4_pm_setup_cpuslocked(void) { consider renaming this to etm4_pm_setup() and handing any cpu locking inside the function. In the circumstances - as part of the driver init rather than probe it may be sufficient to call the cpuhp_setup functions without the _cpuslocked suffix and allow the calls to lock the cpus as they are made. i.e. cpuhp_setup_state_nocalls_cpuslocked() => cpuhp_setup_state_nocalls() Sure, will make this change. int ret; - if (etm4_count++) - return 0; - ret = cpu_pm_register_notifier(_cpu_pm_nb); if (ret) - goto reduce_count; + return ret; ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, "arm/coresight4:starting", @@ -1432,17 +1427,11 @@ static int etm4_pm_setup_cpuslocked(void) unregister_notifier: cpu_pm_unregister_notifier(_cpu_pm_nb); - -reduce_count: - --etm4_count; return ret; } static void etm4_pm_clear(void) { - if (--etm4_count != 0) - return; - cpu_pm_unregister_notifier(_cpu_pm_nb); cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { @@ -1498,22 +1487,12 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (!desc.name) return -ENOMEM; - cpus_read_lock(); etmdrvdata[drvdata->cpu] = drvdata; if (smp_call_function_single(drvdata->cpu, etm4_init_arch_data, drvdata, 1)) dev_err(dev, "ETM arch init failed\n"); - ret = etm4_pm_setup_cpuslocked(); - cpus_read_unlock(); - - /* etm4_pm_setup_cpuslocked() does its own cleanup - exit on error */ - if (ret) { - etmdrvdata[drvdata->cpu] = NULL; - return ret; - } - if (etm4_arch_supported(drvdata->arch) == false) { ret = -EINVAL; goto err_arch_supported; @@ -1560,7 +1539,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) err_arch_supported: etmdrvdata[drvdata->cpu] = NULL; - etm4_pm_clear(); return ret; } @@ -1598,4 +1576,29 @@ static struct amba_driver etm4x_driver = { .probe = etm4_probe, .id_table = etm4_ids, }; -builtin_amba_driver(etm4x_driver); + +static int __init etm4x_init(void) +{ + int ret; + + cpus_read_lock(); + ret = etm4_pm_setup_cpuslocked(); + cpus_read_unlock(); See my comment above about rename and use of cpus_read_lock Yes, thanks for the review Mike. Thanks, Sai -- QUALCOMM INDIA, on behalf of Qualcomm Innovation
Re: [PATCHv2] coresight: etm4x: Fix etm4_count race by moving cpuhp callbacks to init
Hi Sai, On Tue, 28 Jul 2020 at 08:51, Sai Prakash Ranjan wrote: > > etm4_count keeps track of number of ETMv4 registered and on some systems, > a race is observed on etm4_count variable which can lead to multiple calls > to cpuhp_setup_state_nocalls_cpuslocked(). This function internally calls > cpuhp_store_callbacks() which prevents multiple registrations of callbacks > for a given state and due to this race, it returns -EBUSY leading to ETM > probe failures like below. > > coresight-etm4x: probe of 704.etm failed with error -16 > > This race can easily be triggered with async probe by setting probe type > as PROBE_PREFER_ASYNCHRONOUS and with ETM power management property > "arm,coresight-loses-context-with-cpu". > > Prevent this race by moving cpuhp callbacks to etm driver init since the > cpuhp callbacks doesn't have to depend on the etm4_count and can be once > setup during driver init. Similarly we move cpu_pm notifier registration > to driver init and completely remove etm4_count usage. > > Fixes: 9b6a3f3633a5 ("coresight: etmv4: Fix CPU power management setup in > probe() function") > Fixes: 58eb457be028 ("hwtracing/coresight-etm4x: Convert to hotplug state > machine") > Suggested-by: Suzuki K Poulose > Signed-off-by: Sai Prakash Ranjan > --- > > Changes in v2: > * Rearrange cpuhp callbacks and move them to driver init (Suzuki K Poulose) > > --- > drivers/hwtracing/coresight/coresight-etm4x.c | 51 ++- > 1 file changed, 27 insertions(+), 24 deletions(-) > > diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c > b/drivers/hwtracing/coresight/coresight-etm4x.c > index 6d7d2169bfb2..adb71987a1e3 100644 > --- a/drivers/hwtracing/coresight/coresight-etm4x.c > +++ b/drivers/hwtracing/coresight/coresight-etm4x.c > @@ -48,8 +48,6 @@ module_param(pm_save_enable, int, 0444); > MODULE_PARM_DESC(pm_save_enable, > "Save/restore state on power down: 1 = never, 2 = self-hosted"); > > -/* The number of ETMv4 currently registered */ > -static int etm4_count; > static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; > static void etm4_set_default_config(struct etmv4_config *config); > static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, > @@ -1403,12 +1401,9 @@ static int etm4_pm_setup_cpuslocked(void) > { consider renaming this to etm4_pm_setup() and handing any cpu locking inside the function. In the circumstances - as part of the driver init rather than probe it may be sufficient to call the cpuhp_setup functions without the _cpuslocked suffix and allow the calls to lock the cpus as they are made. i.e. cpuhp_setup_state_nocalls_cpuslocked() => cpuhp_setup_state_nocalls() > int ret; > > - if (etm4_count++) > - return 0; > - > ret = cpu_pm_register_notifier(_cpu_pm_nb); > if (ret) > - goto reduce_count; > + return ret; > > ret = > cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, >"arm/coresight4:starting", > @@ -1432,17 +1427,11 @@ static int etm4_pm_setup_cpuslocked(void) > > unregister_notifier: > cpu_pm_unregister_notifier(_cpu_pm_nb); > - > -reduce_count: > - --etm4_count; > return ret; > } > > static void etm4_pm_clear(void) > { > - if (--etm4_count != 0) > - return; > - > cpu_pm_unregister_notifier(_cpu_pm_nb); > cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); > if (hp_online) { > @@ -1498,22 +1487,12 @@ static int etm4_probe(struct amba_device *adev, const > struct amba_id *id) > if (!desc.name) > return -ENOMEM; > > - cpus_read_lock(); > etmdrvdata[drvdata->cpu] = drvdata; > > if (smp_call_function_single(drvdata->cpu, > etm4_init_arch_data, drvdata, 1)) > dev_err(dev, "ETM arch init failed\n"); > > - ret = etm4_pm_setup_cpuslocked(); > - cpus_read_unlock(); > - > - /* etm4_pm_setup_cpuslocked() does its own cleanup - exit on error */ > - if (ret) { > - etmdrvdata[drvdata->cpu] = NULL; > - return ret; > - } > - > if (etm4_arch_supported(drvdata->arch) == false) { > ret = -EINVAL; > goto err_arch_supported; > @@ -1560,7 +1539,6 @@ static int etm4_probe(struct amba_device *adev, const > struct amba_id *id) > > err_arch_supported: > etmdrvdata[drvdata->cpu] = NULL; > - etm4_pm_clear(); > return ret; > } > > @@ -1598,4 +1576,29 @@ static struct amba_driver etm4x_driver = { > .probe = etm4_probe, > .id_table = etm4_ids, > }; > -builtin_amba_driver(etm4x_driver); > + > +static int __init etm4x_init(void) > +{ > + int ret; > + > + cpus_read_lock(); > + ret = etm4_pm_setup_cpuslocked(); > + cpus_read_unlock(); See my comment above about rename and use of
Re: [PATCHv2] coresight: etm4x: Fix etm4_count race by moving cpuhp callbacks to init
On 2020-07-28 13:59, Stephen Boyd wrote: Quoting Sai Prakash Ranjan (2020-07-28 00:51:02) diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c index 6d7d2169bfb2..adb71987a1e3 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.c +++ b/drivers/hwtracing/coresight/coresight-etm4x.c @@ -48,8 +48,6 @@ module_param(pm_save_enable, int, 0444); MODULE_PARM_DESC(pm_save_enable, "Save/restore state on power down: 1 = never, 2 = self-hosted"); -/* The number of ETMv4 currently registered */ -static int etm4_count; static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; static void etm4_set_default_config(struct etmv4_config *config); static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, @@ -1403,12 +1401,9 @@ static int etm4_pm_setup_cpuslocked(void) Is this only called from __init now? If so please mark it as __init then. Yes, will change it. { int ret; - if (etm4_count++) - return 0; - ret = cpu_pm_register_notifier(_cpu_pm_nb); if (ret) - goto reduce_count; + return ret; ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, "arm/coresight4:starting", @@ -1432,17 +1427,11 @@ static int etm4_pm_setup_cpuslocked(void) unregister_notifier: cpu_pm_unregister_notifier(_cpu_pm_nb); - -reduce_count: - --etm4_count; return ret; } static void etm4_pm_clear(void) This is __init too? Will change. { - if (--etm4_count != 0) - return; - cpu_pm_unregister_notifier(_cpu_pm_nb); cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { @@ -1598,4 +1576,29 @@ static struct amba_driver etm4x_driver = { .probe = etm4_probe, .id_table = etm4_ids, }; -builtin_amba_driver(etm4x_driver); + +static int __init etm4x_init(void) +{ + int ret; + + cpus_read_lock(); + ret = etm4_pm_setup_cpuslocked(); + cpus_read_unlock(); + + /* etm4_pm_setup_cpuslocked() does its own cleanup - exit on error */ + if (ret) + return ret; + + ret = amba_driver_register(_driver); + if (ret) { + pr_info("Error registering etm4x driver\n"); Use pr_err() please. Yes indeed, will change. + goto err_init; + } + + return ret; + +err_init: Why is this a goto? + etm4_pm_clear(); + return ret; Instead of just putting this in the if (ret) arm? Will change. +} +module_init(etm4x_init); It was device_initcall before with builtin_amba_driver(), best to not change that. Sure. I will wait to see if there are any more comments on this patch and then post a v3. Thanks for the review Stephen. Thanks, Sai -- QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Re: [PATCHv2] coresight: etm4x: Fix etm4_count race by moving cpuhp callbacks to init
Quoting Sai Prakash Ranjan (2020-07-28 00:51:02) > diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c > b/drivers/hwtracing/coresight/coresight-etm4x.c > index 6d7d2169bfb2..adb71987a1e3 100644 > --- a/drivers/hwtracing/coresight/coresight-etm4x.c > +++ b/drivers/hwtracing/coresight/coresight-etm4x.c > @@ -48,8 +48,6 @@ module_param(pm_save_enable, int, 0444); > MODULE_PARM_DESC(pm_save_enable, > "Save/restore state on power down: 1 = never, 2 = self-hosted"); > > -/* The number of ETMv4 currently registered */ > -static int etm4_count; > static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; > static void etm4_set_default_config(struct etmv4_config *config); > static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, > @@ -1403,12 +1401,9 @@ static int etm4_pm_setup_cpuslocked(void) Is this only called from __init now? If so please mark it as __init then. > { > int ret; > > - if (etm4_count++) > - return 0; > - > ret = cpu_pm_register_notifier(_cpu_pm_nb); > if (ret) > - goto reduce_count; > + return ret; > > ret = > cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, >"arm/coresight4:starting", > @@ -1432,17 +1427,11 @@ static int etm4_pm_setup_cpuslocked(void) > > unregister_notifier: > cpu_pm_unregister_notifier(_cpu_pm_nb); > - > -reduce_count: > - --etm4_count; > return ret; > } > > static void etm4_pm_clear(void) This is __init too? > { > - if (--etm4_count != 0) > - return; > - > cpu_pm_unregister_notifier(_cpu_pm_nb); > cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); > if (hp_online) { > @@ -1598,4 +1576,29 @@ static struct amba_driver etm4x_driver = { > .probe = etm4_probe, > .id_table = etm4_ids, > }; > -builtin_amba_driver(etm4x_driver); > + > +static int __init etm4x_init(void) > +{ > + int ret; > + > + cpus_read_lock(); > + ret = etm4_pm_setup_cpuslocked(); > + cpus_read_unlock(); > + > + /* etm4_pm_setup_cpuslocked() does its own cleanup - exit on error */ > + if (ret) > + return ret; > + > + ret = amba_driver_register(_driver); > + if (ret) { > + pr_info("Error registering etm4x driver\n"); Use pr_err() please. > + goto err_init; > + } > + > + return ret; > + > +err_init: Why is this a goto? > + etm4_pm_clear(); > + return ret; Instead of just putting this in the if (ret) arm? > +} > +module_init(etm4x_init); It was device_initcall before with builtin_amba_driver(), best to not change that.
[PATCHv2] coresight: etm4x: Fix etm4_count race by moving cpuhp callbacks to init
etm4_count keeps track of number of ETMv4 registered and on some systems, a race is observed on etm4_count variable which can lead to multiple calls to cpuhp_setup_state_nocalls_cpuslocked(). This function internally calls cpuhp_store_callbacks() which prevents multiple registrations of callbacks for a given state and due to this race, it returns -EBUSY leading to ETM probe failures like below. coresight-etm4x: probe of 704.etm failed with error -16 This race can easily be triggered with async probe by setting probe type as PROBE_PREFER_ASYNCHRONOUS and with ETM power management property "arm,coresight-loses-context-with-cpu". Prevent this race by moving cpuhp callbacks to etm driver init since the cpuhp callbacks doesn't have to depend on the etm4_count and can be once setup during driver init. Similarly we move cpu_pm notifier registration to driver init and completely remove etm4_count usage. Fixes: 9b6a3f3633a5 ("coresight: etmv4: Fix CPU power management setup in probe() function") Fixes: 58eb457be028 ("hwtracing/coresight-etm4x: Convert to hotplug state machine") Suggested-by: Suzuki K Poulose Signed-off-by: Sai Prakash Ranjan --- Changes in v2: * Rearrange cpuhp callbacks and move them to driver init (Suzuki K Poulose) --- drivers/hwtracing/coresight/coresight-etm4x.c | 51 ++- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c index 6d7d2169bfb2..adb71987a1e3 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.c +++ b/drivers/hwtracing/coresight/coresight-etm4x.c @@ -48,8 +48,6 @@ module_param(pm_save_enable, int, 0444); MODULE_PARM_DESC(pm_save_enable, "Save/restore state on power down: 1 = never, 2 = self-hosted"); -/* The number of ETMv4 currently registered */ -static int etm4_count; static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; static void etm4_set_default_config(struct etmv4_config *config); static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, @@ -1403,12 +1401,9 @@ static int etm4_pm_setup_cpuslocked(void) { int ret; - if (etm4_count++) - return 0; - ret = cpu_pm_register_notifier(_cpu_pm_nb); if (ret) - goto reduce_count; + return ret; ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, "arm/coresight4:starting", @@ -1432,17 +1427,11 @@ static int etm4_pm_setup_cpuslocked(void) unregister_notifier: cpu_pm_unregister_notifier(_cpu_pm_nb); - -reduce_count: - --etm4_count; return ret; } static void etm4_pm_clear(void) { - if (--etm4_count != 0) - return; - cpu_pm_unregister_notifier(_cpu_pm_nb); cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { @@ -1498,22 +1487,12 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (!desc.name) return -ENOMEM; - cpus_read_lock(); etmdrvdata[drvdata->cpu] = drvdata; if (smp_call_function_single(drvdata->cpu, etm4_init_arch_data, drvdata, 1)) dev_err(dev, "ETM arch init failed\n"); - ret = etm4_pm_setup_cpuslocked(); - cpus_read_unlock(); - - /* etm4_pm_setup_cpuslocked() does its own cleanup - exit on error */ - if (ret) { - etmdrvdata[drvdata->cpu] = NULL; - return ret; - } - if (etm4_arch_supported(drvdata->arch) == false) { ret = -EINVAL; goto err_arch_supported; @@ -1560,7 +1539,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) err_arch_supported: etmdrvdata[drvdata->cpu] = NULL; - etm4_pm_clear(); return ret; } @@ -1598,4 +1576,29 @@ static struct amba_driver etm4x_driver = { .probe = etm4_probe, .id_table = etm4_ids, }; -builtin_amba_driver(etm4x_driver); + +static int __init etm4x_init(void) +{ + int ret; + + cpus_read_lock(); + ret = etm4_pm_setup_cpuslocked(); + cpus_read_unlock(); + + /* etm4_pm_setup_cpuslocked() does its own cleanup - exit on error */ + if (ret) + return ret; + + ret = amba_driver_register(_driver); + if (ret) { + pr_info("Error registering etm4x driver\n"); + goto err_init; + } + + return ret; + +err_init: + etm4_pm_clear(); + return ret; +} +module_init(etm4x_init); -- QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation