RE: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
[AMD Public Use]
Some minor things inline
> -Original Message-
> From: amd-gfx On Behalf Of Ramesh
> Errabolu
> Sent: Friday, September 25, 2020 6:03 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Errabolu, Ramesh
> Subject: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to
> get number
> of compute units that are in use.
>
> [Why]
> Allow user to know how many compute units (CU) are in use at any given
> moment.
>
> [How]
> Surface files in Sysfs that allow user to determine the number of compute
> units that are in use for a given process. One Sysfs file is used per
> device.
>
> Signed-off-by: Ramesh Errabolu
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_priv.h| 25 +
> drivers/gpu/drm/amd/amdkfd/kfd_process.c | 68 +++-
> 2 files changed, 92 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 56f92cfff591..3df2b9936458 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -705,6 +705,31 @@ struct kfd_process_device {
>
> struct kobject *kobj_stats;
> unsigned int doorbell_index;
> +
> + /*
> + * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
> + * that is associated with device encoded by "this" struct instance. The
> + * value reflects CU usage by all of the waves launched by this process
> + * on this device. A very important property of occupancy parameter is
> + * that its value is a a snapshot of current use.
is a snapshot
> + *
> + * Following is to be noted regarding how this parameter is reported:
> + *
> + * The number of waves that a CU can launch is limited by couple of
> + * parameters. These are encoded by struct amdgpu_cu_info instance
> + * that is part of every device definition. For GFX9 devices this
> + * translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
> + * do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
> + * when they use. This could change for future devices and therefore
when they do use scratch memory
> + * this example should be considered as a guide.
> + *
> + * All CU's of a device are available for the process. This may not be
> true
> + * under certain conditions - e.g. CU masking.
> + *
> + * Finally number of CU's that are occupied by a process is affected
> by both
> + * number of CU's a device has along with number of other competing
> processes
> + */
> + struct attribute attr_cu_occupancy;
> };
>
> #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index 17d909c86f50..26b716b5eb23 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct
> *work)
> }
> }
>
> +/**
> + * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
> + * by current process. Translates acquired wave count into number of compute
> units
> + * that are occupied.
> + *
> + * @atr: Handle of attribute that allows reporting of wave count. The
> attribute
> + * handle encapsulates GPU device it is associated with, thereby allowing
> collection
> + * of waves in flight, etc
> + *
> + * @buffer: Handle of user provided buffer updated with wave count
> + *
> + * Return: Number of bytes written to user buffer or an error value
> + */
> +static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
> +{
> + int cu_cnt;
> + int wave_cnt;
> + int max_waves_per_cu;
> + struct kfd_dev *dev = NULL;
> + struct kfd_process *proc = NULL;
> + struct kfd_process_device *pdd = NULL;
> +
> + pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
> + dev = pdd->dev;
> + if (dev->kfd2kgd->get_cu_occupancy == NULL)
> + return -EINVAL;
> +
> + cu_cnt = 0;
> + proc = pdd->process;
> + if (pdd->qpd.queue_count == 0) {
> + pr_debug("Gpu-Id: %d has no active queues for process %d\n",
> + dev->id, proc->pasid);
> + return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
> + }
> +
> + /* Collect wave count from device if it supports */
> + wave_cnt = 0;
> + max_waves_per_cu = 0;
> + dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, _cnt,
> + _waves_per_cu);
> +
> + /* Translate wave count to number of compute units */
> + cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
> + return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
> +}
> +
> static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
> char *buffer)
>
[PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h| 25 +
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 68 +++-
2 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 56f92cfff591..3df2b9936458 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -705,6 +705,31 @@ struct kfd_process_device {
struct kobject *kobj_stats;
unsigned int doorbell_index;
+
+ /*
+* @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+* that is associated with device encoded by "this" struct instance. The
+* value reflects CU usage by all of the waves launched by this process
+* on this device. A very important property of occupancy parameter is
+* that its value is a a snapshot of current use.
+*
+* Following is to be noted regarding how this parameter is reported:
+*
+* The number of waves that a CU can launch is limited by couple of
+* parameters. These are encoded by struct amdgpu_cu_info instance
+* that is part of every device definition. For GFX9 devices this
+* translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+* do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+* when they use. This could change for future devices and therefore
+* this example should be considered as a guide.
+*
+* All CU's of a device are available for the process. This may not be
true
+* under certain conditions - e.g. CU masking.
+*
+* Finally number of CU's that are occupied by a process is affected
by both
+* number of CU's a device has along with number of other competing
processes
+*/
+ struct attribute attr_cu_occupancy;
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 17d909c86f50..26b716b5eb23 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct
*work)
}
}
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute
units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing
collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+ int cu_cnt;
+ int wave_cnt;
+ int max_waves_per_cu;
+ struct kfd_dev *dev = NULL;
+ struct kfd_process *proc = NULL;
+ struct kfd_process_device *pdd = NULL;
+
+ pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+ dev = pdd->dev;
+ if (dev->kfd2kgd->get_cu_occupancy == NULL)
+ return -EINVAL;
+
+ cu_cnt = 0;
+ proc = pdd->process;
+ if (pdd->qpd.queue_count == 0) {
+ pr_debug("Gpu-Id: %d has no active queues for process %d\n",
+dev->id, proc->pasid);
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+ }
+
+ /* Collect wave count from device if it supports */
+ wave_cnt = 0;
+ max_waves_per_cu = 0;
+ dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, _cnt,
+ _waves_per_cu);
+
+ /* Translate wave count to number of compute units */
+ cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+}
+
static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
char *buffer)
{
@@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
return 0;
}
+
static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
struct attribute *attr, char *buffer)
{
@@ -359,6 +406,10 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
PAGE_SIZE,
"%llu\n",
jiffies64_to_msecs(evict_jiffies));
+
+ /* Sysfs handle that gets CU occupancy is
Re: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
On 2020-09-17 1:11 p.m., Ramesh Errabolu wrote:
+ if (pdd->qpd.queue_count == 0) {
+ pr_info("%s: Gpu-Id: %d has no active queues for process %d\n",
+ __func__, dev->id, proc->pasid);
pr_debug can be dynamically enabled in dmesg if needed, pr_info will
always show in dmesg.
pr_debug("Gpu-Id: %d has no active queues for process %d\n",
dev->id, proc->pasid);
Philip
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h| 28 ++-
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 101 +++
2 files changed, 114 insertions(+), 15 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 023629f28495..2ce03586d0dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -631,7 +631,7 @@ enum kfd_pdd_bound {
PDD_BOUND_SUSPENDED,
};
-#define MAX_SYSFS_FILENAME_LEN 11
+#define MAX_SYSFS_FILENAME_LEN 32
/*
* SDMA counter runs at 100MHz frequency.
@@ -692,6 +692,32 @@ struct kfd_process_device {
uint64_t sdma_past_activity_counter;
struct attribute attr_sdma;
char sdma_filename[MAX_SYSFS_FILENAME_LEN];
+
+ /*
+* @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+* that is associated with device encoded by "this" struct instance. The
+* value reflects CU usage by all of the waves launched by this process
+* on this device. A very important property of occupancy parameter is
+* that its value is a a snapshot of current use.
+*
+* Following is to be noted regarding how this parameter is reported:
+*
+* The number of waves that a CU can launch is limited by couple of
+* parameters. These are encoded by struct amdgpu_cu_info instance
+* that is part of every device definition. For GFX9 devices this
+* translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+* do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+* when they use. This could change for future devices and therefore
+* this example should be considered as a guide.
+*
+* All CU's of a device are available for the process. This may not be
true
+* under certain conditions - e.g. CU masking.
+*
+* Finally number of CU's that are occupied by a process is affected
by both
+* number of CU's a device has along with number of other competing
processes
+*/
+ struct attribute attr_cu_occupancy;
+ char cu_occupancy_filename[MAX_SYSFS_FILENAME_LEN];
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index a0e12a79ab7d..3baffbc828b2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,63 @@ static void kfd_sdma_activity_worker(struct work_struct
*work)
}
}
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute
units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing
collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+ int cu_cnt;
+ int wave_cnt;
+ int max_waves_per_cu;
+ struct kfd_dev *dev = NULL;
+ struct kfd_process *proc = NULL;
+ struct amdgpu_device *adev = NULL;
+ struct kfd_process_device *pdd = NULL;
+
+ /* Acquire handle of Process-Device-Data associated with attribute */
+ pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+
+ /*
+* Acquire handle of Gpu Device associated with attribute. Determine
+* if ability to collect CU occupancy is defined for this device. If
+* so acquire the handle of process which encapsulates the PASID of
+* process. It is possible that the process might have zero work on
+* device. This is determined by checking if process has any queues
+* @note: Ability to collect is defined only for Gfx9 devices
+*/
+ cu_cnt = 0;
+ dev = pdd->dev;
+ proc = pdd->process;
+ if (pdd->qpd.queue_count == 0) {
+ pr_info("%s: Gpu-Id: %d has no active queues for process %d\n",
+ __func__, dev->id, proc->pasid);
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+ }
+
+ /* Collect wave count from device by reading relevant registers */
+ wave_cnt = 0;
+ if (dev->kfd2kgd->get_cu_occupancy != NULL)
+ dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid,
+
