Re: [RFC v3 2/4] drm/nouveau: Add support for BLCG on Kepler1

[Martin Peres]

On 26/01/18 22:59, Lyude Paul wrote:
> This enables BLCG optimization for kepler1. When using clockgating,
> nvidia's firmware has a set of registers which are initially programmed
> by the vbios with various engine delays and other mysterious settings
> that are safe enough to bring up the GPU. However, the values used by
> the vbios are more power hungry then they need to be, so the nvidia driver

then -> than.

With the comment about not exposing clock gating until patch 2, 3, and 4
have landed addressed, the series is:

Reviewed-by: Martin Peres 

Thanks a lot! I really like how this turned out :)

> writes it's own more optimized set of BLCG settings before enabling
> CG_CTRL. This adds support for programming the optimized BLCG values
> during engine/subdev init, which enables rather significant power
> savings.
> 
> This introduces the nvkm_therm_clkgate_init() helper, which we use to
> program the optimized BLCG settings before enabling clockgating with
> nvkm_therm_clkgate_enable.
> 
> As well, this commit shares a lot more code with Fermi since BLCG is
> mostly the same there as far as we can tell. In the future, it's likely
> we'll reformat the clkgate_packs for kepler1 so that they share a list
> of mmio packs with Fermi.
> 
> Signed-off-by: Lyude Paul 
> ---
>  .../gpu/drm/nouveau/include/nvkm/subdev/therm.h|  12 ++
>  drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h |   1 +
>  drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 207 
> +
>  drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h |  55 ++
>  drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c |   6 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c |  47 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h |  35 
>  drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h  |   2 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild   |   1 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c   |  10 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c  |  75 
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c  |   1 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c  |   2 +-
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h   |   8 +
>  14 files changed, 461 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h
>  create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h
>  create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c
> 
> diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h 
> b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
> index 240b19bb4667..9398d9f09339 100644
> --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
> +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
> @@ -46,6 +46,16 @@ enum nvkm_therm_attr_type {
>   NVKM_THERM_ATTR_THRS_SHUTDOWN_HYST = 17,
>  };
>  
> +struct nvkm_therm_clkgate_init {
> + u32 addr;
> + u8  count;
> + u32 data;
> +};
> +
> +struct nvkm_therm_clkgate_pack {
> + const struct nvkm_therm_clkgate_init *init;
> +};
> +
>  struct nvkm_therm {
>   const struct nvkm_therm_func *func;
>   struct nvkm_subdev subdev;
> @@ -92,6 +102,8 @@ struct nvkm_therm {
>  int nvkm_therm_temp_get(struct nvkm_therm *);
>  int nvkm_therm_fan_sense(struct nvkm_therm *);
>  int nvkm_therm_cstate(struct nvkm_therm *, int, int);
> +void nvkm_therm_clkgate_init(struct nvkm_therm *,
> +  const struct nvkm_therm_clkgate_pack *);
>  void nvkm_therm_clkgate_enable(struct nvkm_therm *);
>  void nvkm_therm_clkgate_fini(struct nvkm_therm *, bool);
>  
> diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h 
> b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
> index d7c2adb9b543..c8ec3fd97155 100644
> --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
> +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
> @@ -137,6 +137,7 @@ struct gf100_gr_func {
>   int (*rops)(struct gf100_gr *);
>   int ppc_nr;
>   const struct gf100_grctx_func *grctx;
> + const struct nvkm_therm_clkgate_pack *clkgate_pack;
>   struct nvkm_sclass sclass[];
>  };
>  
> diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c 
> b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
> index 5e82f94c2245..17cea9c70f7f 100644
> --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
> +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
> @@ -22,6 +22,7 @@
>   * Authors: Ben Skeggs 
>   */
>  #include "gf100.h"
> +#include "gk104.h"
>  #include "ctxgf100.h"
>  
>  #include 
> @@ -173,6 +174,208 @@ gk104_gr_pack_mmio[] = {
>   {}
>  };
>  
> +const struct nvkm_therm_clkgate_init
> +gk104_clkgate_blcg_init_main_0[] = {
> + { 0x4041f0, 1, 0x4046 },
> + { 0x409890, 1, 0x0045 },
> + { 0x4098b0, 1, 0x007f },
> + {}
> +};
> +
> +const struct nvkm_therm_clkgate_init
> +gk104_clkgate_blcg_init_rstr2d_0[] = {
> + { 0x4078c0, 1, 0x0042 },
> 

[RFC v3 2/4] drm/nouveau: Add support for BLCG on Kepler1

[Lyude Paul]

This enables BLCG optimization for kepler1. When using clockgating,
nvidia's firmware has a set of registers which are initially programmed
by the vbios with various engine delays and other mysterious settings
that are safe enough to bring up the GPU. However, the values used by
the vbios are more power hungry then they need to be, so the nvidia driver
writes it's own more optimized set of BLCG settings before enabling
CG_CTRL. This adds support for programming the optimized BLCG values
during engine/subdev init, which enables rather significant power
savings.

This introduces the nvkm_therm_clkgate_init() helper, which we use to
program the optimized BLCG settings before enabling clockgating with
nvkm_therm_clkgate_enable.

As well, this commit shares a lot more code with Fermi since BLCG is
mostly the same there as far as we can tell. In the future, it's likely
we'll reformat the clkgate_packs for kepler1 so that they share a list
of mmio packs with Fermi.

Signed-off-by: Lyude Paul 
---
 .../gpu/drm/nouveau/include/nvkm/subdev/therm.h|  12 ++
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h |   1 +
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 207 +
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h |  55 ++
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c |   6 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c |  47 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h |  35 
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h  |   2 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild   |   1 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c   |  10 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c  |  75 
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c  |   1 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c  |   2 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h   |   8 +
 14 files changed, 461 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h 
b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
index 240b19bb4667..9398d9f09339 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
@@ -46,6 +46,16 @@ enum nvkm_therm_attr_type {
NVKM_THERM_ATTR_THRS_SHUTDOWN_HYST = 17,
 };
 
+struct nvkm_therm_clkgate_init {
+   u32 addr;
+   u8  count;
+   u32 data;
+};
+
+struct nvkm_therm_clkgate_pack {
+   const struct nvkm_therm_clkgate_init *init;
+};
+
 struct nvkm_therm {
const struct nvkm_therm_func *func;
struct nvkm_subdev subdev;
@@ -92,6 +102,8 @@ struct nvkm_therm {
 int nvkm_therm_temp_get(struct nvkm_therm *);
 int nvkm_therm_fan_sense(struct nvkm_therm *);
 int nvkm_therm_cstate(struct nvkm_therm *, int, int);
+void nvkm_therm_clkgate_init(struct nvkm_therm *,
+const struct nvkm_therm_clkgate_pack *);
 void nvkm_therm_clkgate_enable(struct nvkm_therm *);
 void nvkm_therm_clkgate_fini(struct nvkm_therm *, bool);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h 
b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
index d7c2adb9b543..c8ec3fd97155 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
@@ -137,6 +137,7 @@ struct gf100_gr_func {
int (*rops)(struct gf100_gr *);
int ppc_nr;
const struct gf100_grctx_func *grctx;
+   const struct nvkm_therm_clkgate_pack *clkgate_pack;
struct nvkm_sclass sclass[];
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c 
b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
index 5e82f94c2245..17cea9c70f7f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
@@ -22,6 +22,7 @@
  * Authors: Ben Skeggs 
  */
 #include "gf100.h"
+#include "gk104.h"
 #include "ctxgf100.h"
 
 #include 
@@ -173,6 +174,208 @@ gk104_gr_pack_mmio[] = {
{}
 };
 
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_main_0[] = {
+   { 0x4041f0, 1, 0x4046 },
+   { 0x409890, 1, 0x0045 },
+   { 0x4098b0, 1, 0x007f },
+   {}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_rstr2d_0[] = {
+   { 0x4078c0, 1, 0x0042 },
+   {}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_unk_0[] = {
+   { 0x406000, 1, 0x4044 },
+   { 0x405860, 1, 0x4042 },
+   { 0x40590c, 1, 0x4042 },
+   {}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gcc_0[] = {
+   { 0x408040, 1, 0x4044 },
+   {}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_sked_0[] = {
+   { 0x407000, 1,