Re: [PATCH v3 05/10] powerpc/pseries/vas: Close windows with DLPAR core removal
On Mon, 2022-02-14 at 13:17 +1000, Nicholas Piggin wrote:
> Excerpts from Haren Myneni's message of January 22, 2022 5:57 am:
> > The hypervisor reduces the available credits if the core is removed
> > from the LPAR. So there is possibility of using excessive credits
> > (windows) in the LPAR and the hypervisor expects the system to
> > close
> > the excessive windows. Even though the user space can continue to
> > use
> > these windows to send compression requests to NX, the hypervisor
> > expects
> > the LPAR to reduce these windows usage so that NX load can be
> > equally
> > distributed across all LPARs in the system.
> >
> > When the DLPAR notifier is received, get the new VAS capabilities
> > from
> > the hypervisor and close the excessive windows in the hypervisor.
> > Also
> > the kernel unmaps the paste address so that the user space receives
> > paste
> > failure until these windows are active with the later DLPAR (core
> > add).
>
> The changelog needs work. Unmapping the window and the ramifications
> of
> that needs more description here or in comments.
Thanks will change.
>
> > Signed-off-by: Haren Myneni
> > ---
> > arch/powerpc/include/asm/vas.h | 1 +
> > arch/powerpc/platforms/book3s/vas-api.c | 2 +
> > arch/powerpc/platforms/pseries/vas.c| 117
> > ++--
> > arch/powerpc/platforms/pseries/vas.h| 1 +
> > 4 files changed, 112 insertions(+), 9 deletions(-)
> >
> > diff --git a/arch/powerpc/include/asm/vas.h
> > b/arch/powerpc/include/asm/vas.h
> > index f1efe86563cc..ddc05a8fc2e3 100644
> > --- a/arch/powerpc/include/asm/vas.h
> > +++ b/arch/powerpc/include/asm/vas.h
> > @@ -74,6 +74,7 @@ struct vas_user_win_ref {
> > struct mm_struct *mm; /* Linux process mm_struct */
> > struct mutex mmap_mutex;/* protects paste address mmap() */
> > /* with DLPAR close/open
> > windows */
> > + struct vm_area_struct *vma; /* Save VMA and used in DLPAR ops
> > */
> > };
> >
> > /*
> > diff --git a/arch/powerpc/platforms/book3s/vas-api.c
> > b/arch/powerpc/platforms/book3s/vas-api.c
> > index 2b0ced611f32..a63fd48e34a7 100644
> > --- a/arch/powerpc/platforms/book3s/vas-api.c
> > +++ b/arch/powerpc/platforms/book3s/vas-api.c
> > @@ -399,6 +399,8 @@ static int coproc_mmap(struct file *fp, struct
> > vm_area_struct *vma)
> > pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
> > paste_addr, vma->vm_start, rc);
> >
> > + txwin->task_ref.vma = vma;
> > +
> > return rc;
> > }
> >
> > diff --git a/arch/powerpc/platforms/pseries/vas.c
> > b/arch/powerpc/platforms/pseries/vas.c
> > index d9ff73d7704d..75ccd0a599ec 100644
> > --- a/arch/powerpc/platforms/pseries/vas.c
> > +++ b/arch/powerpc/platforms/pseries/vas.c
> > @@ -370,13 +370,28 @@ static struct vas_window
> > *vas_allocate_window(int vas_id, u64 flags,
> > if (rc)
> > goto out_free;
> >
> > - vas_user_win_add_mm_context(>vas_win.task_ref);
> > txwin->win_type = cop_feat_caps->win_type;
> > mutex_lock(_pseries_mutex);
> > - list_add(>win_list, >list);
> > + /*
> > +* Possible to loose the acquired credit with DLPAR core
>
> s/loose/lose/g
>
> > +* removal after the window is opened. So if there are any
> > +* closed windows (means with lost credits), do not give new
> > +* window to user space. New windows will be opened only
> > +* after the existing windows are reopened when credits are
> > +* available.
> > +*/
> > + if (!caps->close_wins) {
> > + list_add(>win_list, >list);
> > + caps->num_wins++;
> > + mutex_unlock(_pseries_mutex);
> > + vas_user_win_add_mm_context(>vas_win.task_ref);
> > + return >vas_win;
> > + }
> > mutex_unlock(_pseries_mutex);
> >
> > - return >vas_win;
> > + put_vas_user_win_ref(>vas_win.task_ref);
> > + rc = -EBUSY;
> > + pr_err("No credit is available to allocate window\n");
> >
> > out_free:
> > /*
> > @@ -439,14 +454,24 @@ static int vas_deallocate_window(struct
> > vas_window *vwin)
> >
> > caps = [win->win_type].caps;
> > mutex_lock(_pseries_mutex);
> > - rc = deallocate_free_window(win);
> > - if (rc) {
> > - mutex_unlock(_pseries_mutex);
> > - return rc;
> > - }
> > + /*
> > +* VAS window is already closed in the hypervisor when
> > +* lost the credit. So just remove the entry from
> > +* the list, remove task references and free vas_window
> > +* struct.
> > +*/
> > + if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) {
> > + rc = deallocate_free_window(win);
> > + if (rc) {
> > + mutex_unlock(_pseries_mutex);
> > + return rc;
> > + }
> > + } else
> > + vascaps[win->win_type].close_wins--;
> >
> > list_del(>win_list);
> > atomic_dec(>used_creds);
> > +
Re: [PATCH v3 05/10] powerpc/pseries/vas: Close windows with DLPAR core removal
Excerpts from Haren Myneni's message of January 22, 2022 5:57 am:
>
> The hypervisor reduces the available credits if the core is removed
> from the LPAR. So there is possibility of using excessive credits
> (windows) in the LPAR and the hypervisor expects the system to close
> the excessive windows. Even though the user space can continue to use
> these windows to send compression requests to NX, the hypervisor expects
> the LPAR to reduce these windows usage so that NX load can be equally
> distributed across all LPARs in the system.
>
> When the DLPAR notifier is received, get the new VAS capabilities from
> the hypervisor and close the excessive windows in the hypervisor. Also
> the kernel unmaps the paste address so that the user space receives paste
> failure until these windows are active with the later DLPAR (core add).
The changelog needs work. Unmapping the window and the ramifications of
that needs more description here or in comments.
>
> Signed-off-by: Haren Myneni
> ---
> arch/powerpc/include/asm/vas.h | 1 +
> arch/powerpc/platforms/book3s/vas-api.c | 2 +
> arch/powerpc/platforms/pseries/vas.c| 117 ++--
> arch/powerpc/platforms/pseries/vas.h| 1 +
> 4 files changed, 112 insertions(+), 9 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
> index f1efe86563cc..ddc05a8fc2e3 100644
> --- a/arch/powerpc/include/asm/vas.h
> +++ b/arch/powerpc/include/asm/vas.h
> @@ -74,6 +74,7 @@ struct vas_user_win_ref {
> struct mm_struct *mm; /* Linux process mm_struct */
> struct mutex mmap_mutex;/* protects paste address mmap() */
> /* with DLPAR close/open windows */
> + struct vm_area_struct *vma; /* Save VMA and used in DLPAR ops */
> };
>
> /*
> diff --git a/arch/powerpc/platforms/book3s/vas-api.c
> b/arch/powerpc/platforms/book3s/vas-api.c
> index 2b0ced611f32..a63fd48e34a7 100644
> --- a/arch/powerpc/platforms/book3s/vas-api.c
> +++ b/arch/powerpc/platforms/book3s/vas-api.c
> @@ -399,6 +399,8 @@ static int coproc_mmap(struct file *fp, struct
> vm_area_struct *vma)
> pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
> paste_addr, vma->vm_start, rc);
>
> + txwin->task_ref.vma = vma;
> +
> return rc;
> }
>
> diff --git a/arch/powerpc/platforms/pseries/vas.c
> b/arch/powerpc/platforms/pseries/vas.c
> index d9ff73d7704d..75ccd0a599ec 100644
> --- a/arch/powerpc/platforms/pseries/vas.c
> +++ b/arch/powerpc/platforms/pseries/vas.c
> @@ -370,13 +370,28 @@ static struct vas_window *vas_allocate_window(int
> vas_id, u64 flags,
> if (rc)
> goto out_free;
>
> - vas_user_win_add_mm_context(>vas_win.task_ref);
> txwin->win_type = cop_feat_caps->win_type;
> mutex_lock(_pseries_mutex);
> - list_add(>win_list, >list);
> + /*
> + * Possible to loose the acquired credit with DLPAR core
s/loose/lose/g
> + * removal after the window is opened. So if there are any
> + * closed windows (means with lost credits), do not give new
> + * window to user space. New windows will be opened only
> + * after the existing windows are reopened when credits are
> + * available.
> + */
> + if (!caps->close_wins) {
> + list_add(>win_list, >list);
> + caps->num_wins++;
> + mutex_unlock(_pseries_mutex);
> + vas_user_win_add_mm_context(>vas_win.task_ref);
> + return >vas_win;
> + }
> mutex_unlock(_pseries_mutex);
>
> - return >vas_win;
> + put_vas_user_win_ref(>vas_win.task_ref);
> + rc = -EBUSY;
> + pr_err("No credit is available to allocate window\n");
>
> out_free:
> /*
> @@ -439,14 +454,24 @@ static int vas_deallocate_window(struct vas_window
> *vwin)
>
> caps = [win->win_type].caps;
> mutex_lock(_pseries_mutex);
> - rc = deallocate_free_window(win);
> - if (rc) {
> - mutex_unlock(_pseries_mutex);
> - return rc;
> - }
> + /*
> + * VAS window is already closed in the hypervisor when
> + * lost the credit. So just remove the entry from
> + * the list, remove task references and free vas_window
> + * struct.
> + */
> + if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) {
> + rc = deallocate_free_window(win);
> + if (rc) {
> + mutex_unlock(_pseries_mutex);
> + return rc;
> + }
> + } else
> + vascaps[win->win_type].close_wins--;
>
> list_del(>win_list);
> atomic_dec(>used_creds);
> + vascaps[win->win_type].num_wins--;
> mutex_unlock(_pseries_mutex);
>
> put_vas_user_win_ref(>task_ref);
> @@ -622,6 +647,72 @@ static int reconfig_open_windows(struct vas_caps *vcaps,
> int creds)
> return rc;
> }
>
> +/*
> + * The hypervisor reduces
[PATCH v3 05/10] powerpc/pseries/vas: Close windows with DLPAR core removal
The hypervisor reduces the available credits if the core is removed
from the LPAR. So there is possibility of using excessive credits
(windows) in the LPAR and the hypervisor expects the system to close
the excessive windows. Even though the user space can continue to use
these windows to send compression requests to NX, the hypervisor expects
the LPAR to reduce these windows usage so that NX load can be equally
distributed across all LPARs in the system.
When the DLPAR notifier is received, get the new VAS capabilities from
the hypervisor and close the excessive windows in the hypervisor. Also
the kernel unmaps the paste address so that the user space receives paste
failure until these windows are active with the later DLPAR (core add).
Signed-off-by: Haren Myneni
---
arch/powerpc/include/asm/vas.h | 1 +
arch/powerpc/platforms/book3s/vas-api.c | 2 +
arch/powerpc/platforms/pseries/vas.c| 117 ++--
arch/powerpc/platforms/pseries/vas.h| 1 +
4 files changed, 112 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index f1efe86563cc..ddc05a8fc2e3 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -74,6 +74,7 @@ struct vas_user_win_ref {
struct mm_struct *mm; /* Linux process mm_struct */
struct mutex mmap_mutex;/* protects paste address mmap() */
/* with DLPAR close/open windows */
+ struct vm_area_struct *vma; /* Save VMA and used in DLPAR ops */
};
/*
diff --git a/arch/powerpc/platforms/book3s/vas-api.c
b/arch/powerpc/platforms/book3s/vas-api.c
index 2b0ced611f32..a63fd48e34a7 100644
--- a/arch/powerpc/platforms/book3s/vas-api.c
+++ b/arch/powerpc/platforms/book3s/vas-api.c
@@ -399,6 +399,8 @@ static int coproc_mmap(struct file *fp, struct
vm_area_struct *vma)
pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
paste_addr, vma->vm_start, rc);
+ txwin->task_ref.vma = vma;
+
return rc;
}
diff --git a/arch/powerpc/platforms/pseries/vas.c
b/arch/powerpc/platforms/pseries/vas.c
index d9ff73d7704d..75ccd0a599ec 100644
--- a/arch/powerpc/platforms/pseries/vas.c
+++ b/arch/powerpc/platforms/pseries/vas.c
@@ -370,13 +370,28 @@ static struct vas_window *vas_allocate_window(int vas_id,
u64 flags,
if (rc)
goto out_free;
- vas_user_win_add_mm_context(>vas_win.task_ref);
txwin->win_type = cop_feat_caps->win_type;
mutex_lock(_pseries_mutex);
- list_add(>win_list, >list);
+ /*
+* Possible to loose the acquired credit with DLPAR core
+* removal after the window is opened. So if there are any
+* closed windows (means with lost credits), do not give new
+* window to user space. New windows will be opened only
+* after the existing windows are reopened when credits are
+* available.
+*/
+ if (!caps->close_wins) {
+ list_add(>win_list, >list);
+ caps->num_wins++;
+ mutex_unlock(_pseries_mutex);
+ vas_user_win_add_mm_context(>vas_win.task_ref);
+ return >vas_win;
+ }
mutex_unlock(_pseries_mutex);
- return >vas_win;
+ put_vas_user_win_ref(>vas_win.task_ref);
+ rc = -EBUSY;
+ pr_err("No credit is available to allocate window\n");
out_free:
/*
@@ -439,14 +454,24 @@ static int vas_deallocate_window(struct vas_window *vwin)
caps = [win->win_type].caps;
mutex_lock(_pseries_mutex);
- rc = deallocate_free_window(win);
- if (rc) {
- mutex_unlock(_pseries_mutex);
- return rc;
- }
+ /*
+* VAS window is already closed in the hypervisor when
+* lost the credit. So just remove the entry from
+* the list, remove task references and free vas_window
+* struct.
+*/
+ if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) {
+ rc = deallocate_free_window(win);
+ if (rc) {
+ mutex_unlock(_pseries_mutex);
+ return rc;
+ }
+ } else
+ vascaps[win->win_type].close_wins--;
list_del(>win_list);
atomic_dec(>used_creds);
+ vascaps[win->win_type].num_wins--;
mutex_unlock(_pseries_mutex);
put_vas_user_win_ref(>task_ref);
@@ -622,6 +647,72 @@ static int reconfig_open_windows(struct vas_caps *vcaps,
int creds)
return rc;
}
+/*
+ * The hypervisor reduces the available credits if the LPAR lost core. It
+ * means the excessive windows should not be active and the user space
+ * should not be using these windows to send compression requests to NX.
+ * So the kernel closes the excessive windows and unmap the paste address
+ * such that the user space receives paste instruction
