Re: [Qemu-devel] [PATCH v3 03/29] postcopy: use UFFDIO_ZEROPAGE only when available

[Dr. David Alan Gilbert]

* Peter Xu (pet...@redhat.com) wrote:
> On Fri, Feb 16, 2018 at 01:15:59PM +, Dr. David Alan Gilbert (git) wrote:
> > From: "Dr. David Alan Gilbert" 
> > 
> > Use a flag on the RAMBlock to state whether it has the
> > UFFDIO_ZEROPAGE capability, use it when it's available.
> > 
> > This allows the use of postcopy on tmpfs as well as hugepage
> > backed files.
> > 
> > Signed-off-by: Dr. David Alan Gilbert 
> > ---
> >  exec.c| 15 +++
> >  include/exec/cpu-common.h |  3 +++
> >  migration/postcopy-ram.c  | 13 ++---
> >  3 files changed, 28 insertions(+), 3 deletions(-)
> > 
> > diff --git a/exec.c b/exec.c
> > index 0ec73bc917..1dc15298c2 100644
> > --- a/exec.c
> > +++ b/exec.c
> > @@ -99,6 +99,11 @@ static MemoryRegion io_mem_unassigned;
> >   */
> >  #define RAM_RESIZEABLE (1 << 2)
> >  
> > +/* UFFDIO_ZEROPAGE is available on this RAMBlock to atomically
> > + * zero the page and wake waiting processes.
> > + * (Set during postcopy)
> > + */
> > +#define RAM_UF_ZEROPAGE (1 << 3)
> >  #endif
> >  
> >  #ifdef TARGET_PAGE_BITS_VARY
> > @@ -1767,6 +1772,16 @@ bool qemu_ram_is_shared(RAMBlock *rb)
> >  return rb->flags & RAM_SHARED;
> >  }
> >  
> > +bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
> > +{
> > +return rb->flags & RAM_UF_ZEROPAGE;
> > +}
> > +
> > +void qemu_ram_set_uf_zeroable(RAMBlock *rb)
> > +{
> > +rb->flags |= RAM_UF_ZEROPAGE;
> > +}
> > +
> >  /* Called with iothread lock held.  */
> >  void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState 
> > *dev)
> >  {
> > diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
> > index 0d861a6289..24d335f95d 100644
> > --- a/include/exec/cpu-common.h
> > +++ b/include/exec/cpu-common.h
> > @@ -73,6 +73,9 @@ void qemu_ram_set_idstr(RAMBlock *block, const char 
> > *name, DeviceState *dev);
> >  void qemu_ram_unset_idstr(RAMBlock *block);
> >  const char *qemu_ram_get_idstr(RAMBlock *rb);
> >  bool qemu_ram_is_shared(RAMBlock *rb);
> > +bool qemu_ram_is_uf_zeroable(RAMBlock *rb);
> > +void qemu_ram_set_uf_zeroable(RAMBlock *rb);
> > +
> >  size_t qemu_ram_pagesize(RAMBlock *block);
> >  size_t qemu_ram_pagesize_largest(void);
> >  
> > diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
> > index bec6c2c66b..6297979700 100644
> > --- a/migration/postcopy-ram.c
> > +++ b/migration/postcopy-ram.c
> > @@ -490,6 +490,10 @@ static int ram_block_enable_notify(const char 
> > *block_name, void *host_addr,
> >  error_report("%s userfault: Region doesn't support COPY", 
> > __func__);
> >  return -1;
> >  }
> > +if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) {
> > +RAMBlock *rb = qemu_ram_block_by_name(block_name);
> > +qemu_ram_set_uf_zeroable(rb);
> > +}
> 
> So the zeroable flag is only set after a listening operation of
> postcopy migration.  One thing I am a bit worried is that if someone
> else wants to use the flag for a RAMBlock he/she may not notice this.
> Say, qemu_ram_is_uf_zeroable() is not valid if there is no such an
> incoming postcopy migration.

Yes, the problem is you don't get to know until you register it with
userfault.

> Maybe worth add a comment in the flag definition about this?

Yes, I've added:
/* Note: Only set at the start of postcopy */

above the qemu_ram_is_uf_zeroable() defintiion; also see the existing
'(Set during postcopy)' comment above the #define.


> Not a big deal (considering that I see no potential QEMU user for
> userfaultfd in short peroid), so no matter what:
> 
> Reviewed-by: Peter Xu 

Thanks.

Dave

> 
> >  
> >  return 0;
> >  }
> > @@ -699,11 +703,14 @@ int postcopy_place_page(MigrationIncomingState *mis, 
> > void *host, void *from,
> >  int postcopy_place_page_zero(MigrationIncomingState *mis, void *host,
> >   RAMBlock *rb)
> >  {
> > +size_t pagesize = qemu_ram_pagesize(rb);
> >  trace_postcopy_place_page_zero(host);
> >  
> > -if (qemu_ram_pagesize(rb) == getpagesize()) {
> > -if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, 
> > getpagesize(),
> > -rb)) {
> > +/* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE
> > + * but it's not available for everything (e.g. hugetlbpages)
> > + */
> > +if (qemu_ram_is_uf_zeroable(rb)) {
> > +if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, pagesize, 
> > rb)) {
> >  int e = errno;
> >  error_report("%s: %s zero host: %p",
> >   __func__, strerror(e), host);
> > -- 
> > 2.14.3
> > 
> 
> -- 
> Peter Xu
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK

Re: [Qemu-devel] [PATCH v3 03/29] postcopy: use UFFDIO_ZEROPAGE only when available

[Peter Xu]

On Fri, Feb 16, 2018 at 01:15:59PM +, Dr. David Alan Gilbert (git) wrote:
> From: "Dr. David Alan Gilbert" 
> 
> Use a flag on the RAMBlock to state whether it has the
> UFFDIO_ZEROPAGE capability, use it when it's available.
> 
> This allows the use of postcopy on tmpfs as well as hugepage
> backed files.
> 
> Signed-off-by: Dr. David Alan Gilbert 
> ---
>  exec.c| 15 +++
>  include/exec/cpu-common.h |  3 +++
>  migration/postcopy-ram.c  | 13 ++---
>  3 files changed, 28 insertions(+), 3 deletions(-)
> 
> diff --git a/exec.c b/exec.c
> index 0ec73bc917..1dc15298c2 100644
> --- a/exec.c
> +++ b/exec.c
> @@ -99,6 +99,11 @@ static MemoryRegion io_mem_unassigned;
>   */
>  #define RAM_RESIZEABLE (1 << 2)
>  
> +/* UFFDIO_ZEROPAGE is available on this RAMBlock to atomically
> + * zero the page and wake waiting processes.
> + * (Set during postcopy)
> + */
> +#define RAM_UF_ZEROPAGE (1 << 3)
>  #endif
>  
>  #ifdef TARGET_PAGE_BITS_VARY
> @@ -1767,6 +1772,16 @@ bool qemu_ram_is_shared(RAMBlock *rb)
>  return rb->flags & RAM_SHARED;
>  }
>  
> +bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
> +{
> +return rb->flags & RAM_UF_ZEROPAGE;
> +}
> +
> +void qemu_ram_set_uf_zeroable(RAMBlock *rb)
> +{
> +rb->flags |= RAM_UF_ZEROPAGE;
> +}
> +
>  /* Called with iothread lock held.  */
>  void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState 
> *dev)
>  {
> diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
> index 0d861a6289..24d335f95d 100644
> --- a/include/exec/cpu-common.h
> +++ b/include/exec/cpu-common.h
> @@ -73,6 +73,9 @@ void qemu_ram_set_idstr(RAMBlock *block, const char *name, 
> DeviceState *dev);
>  void qemu_ram_unset_idstr(RAMBlock *block);
>  const char *qemu_ram_get_idstr(RAMBlock *rb);
>  bool qemu_ram_is_shared(RAMBlock *rb);
> +bool qemu_ram_is_uf_zeroable(RAMBlock *rb);
> +void qemu_ram_set_uf_zeroable(RAMBlock *rb);
> +
>  size_t qemu_ram_pagesize(RAMBlock *block);
>  size_t qemu_ram_pagesize_largest(void);
>  
> diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
> index bec6c2c66b..6297979700 100644
> --- a/migration/postcopy-ram.c
> +++ b/migration/postcopy-ram.c
> @@ -490,6 +490,10 @@ static int ram_block_enable_notify(const char 
> *block_name, void *host_addr,
>  error_report("%s userfault: Region doesn't support COPY", __func__);
>  return -1;
>  }
> +if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) {
> +RAMBlock *rb = qemu_ram_block_by_name(block_name);
> +qemu_ram_set_uf_zeroable(rb);
> +}

So the zeroable flag is only set after a listening operation of
postcopy migration.  One thing I am a bit worried is that if someone
else wants to use the flag for a RAMBlock he/she may not notice this.
Say, qemu_ram_is_uf_zeroable() is not valid if there is no such an
incoming postcopy migration.

Maybe worth add a comment in the flag definition about this?

Not a big deal (considering that I see no potential QEMU user for
userfaultfd in short peroid), so no matter what:

Reviewed-by: Peter Xu 

>  
>  return 0;
>  }
> @@ -699,11 +703,14 @@ int postcopy_place_page(MigrationIncomingState *mis, 
> void *host, void *from,
>  int postcopy_place_page_zero(MigrationIncomingState *mis, void *host,
>   RAMBlock *rb)
>  {
> +size_t pagesize = qemu_ram_pagesize(rb);
>  trace_postcopy_place_page_zero(host);
>  
> -if (qemu_ram_pagesize(rb) == getpagesize()) {
> -if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, getpagesize(),
> -rb)) {
> +/* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE
> + * but it's not available for everything (e.g. hugetlbpages)
> + */
> +if (qemu_ram_is_uf_zeroable(rb)) {
> +if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, pagesize, 
> rb)) {
>  int e = errno;
>  error_report("%s: %s zero host: %p",
>   __func__, strerror(e), host);
> -- 
> 2.14.3
> 

-- 
Peter Xu

[Qemu-devel] [PATCH v3 03/29] postcopy: use UFFDIO_ZEROPAGE only when available

[Dr. David Alan Gilbert (git)]

From: "Dr. David Alan Gilbert" 

Use a flag on the RAMBlock to state whether it has the
UFFDIO_ZEROPAGE capability, use it when it's available.

This allows the use of postcopy on tmpfs as well as hugepage
backed files.

Signed-off-by: Dr. David Alan Gilbert 
---
 exec.c| 15 +++
 include/exec/cpu-common.h |  3 +++
 migration/postcopy-ram.c  | 13 ++---
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/exec.c b/exec.c
index 0ec73bc917..1dc15298c2 100644
--- a/exec.c
+++ b/exec.c
@@ -99,6 +99,11 @@ static MemoryRegion io_mem_unassigned;
  */
 #define RAM_RESIZEABLE (1 << 2)
 
+/* UFFDIO_ZEROPAGE is available on this RAMBlock to atomically
+ * zero the page and wake waiting processes.
+ * (Set during postcopy)
+ */
+#define RAM_UF_ZEROPAGE (1 << 3)
 #endif
 
 #ifdef TARGET_PAGE_BITS_VARY
@@ -1767,6 +1772,16 @@ bool qemu_ram_is_shared(RAMBlock *rb)
 return rb->flags & RAM_SHARED;
 }
 
+bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
+{
+return rb->flags & RAM_UF_ZEROPAGE;
+}
+
+void qemu_ram_set_uf_zeroable(RAMBlock *rb)
+{
+rb->flags |= RAM_UF_ZEROPAGE;
+}
+
 /* Called with iothread lock held.  */
 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState 
*dev)
 {
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index 0d861a6289..24d335f95d 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -73,6 +73,9 @@ void qemu_ram_set_idstr(RAMBlock *block, const char *name, 
DeviceState *dev);
 void qemu_ram_unset_idstr(RAMBlock *block);
 const char *qemu_ram_get_idstr(RAMBlock *rb);
 bool qemu_ram_is_shared(RAMBlock *rb);
+bool qemu_ram_is_uf_zeroable(RAMBlock *rb);
+void qemu_ram_set_uf_zeroable(RAMBlock *rb);
+
 size_t qemu_ram_pagesize(RAMBlock *block);
 size_t qemu_ram_pagesize_largest(void);
 
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
index bec6c2c66b..6297979700 100644
--- a/migration/postcopy-ram.c
+++ b/migration/postcopy-ram.c
@@ -490,6 +490,10 @@ static int ram_block_enable_notify(const char *block_name, 
void *host_addr,
 error_report("%s userfault: Region doesn't support COPY", __func__);
 return -1;
 }
+if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) {
+RAMBlock *rb = qemu_ram_block_by_name(block_name);
+qemu_ram_set_uf_zeroable(rb);
+}
 
 return 0;
 }
@@ -699,11 +703,14 @@ int postcopy_place_page(MigrationIncomingState *mis, void 
*host, void *from,
 int postcopy_place_page_zero(MigrationIncomingState *mis, void *host,
  RAMBlock *rb)
 {
+size_t pagesize = qemu_ram_pagesize(rb);
 trace_postcopy_place_page_zero(host);
 
-if (qemu_ram_pagesize(rb) == getpagesize()) {
-if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, getpagesize(),
-rb)) {
+/* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE
+ * but it's not available for everything (e.g. hugetlbpages)
+ */
+if (qemu_ram_is_uf_zeroable(rb)) {
+if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, pagesize, rb)) {
 int e = errno;
 error_report("%s: %s zero host: %p",
  __func__, strerror(e), host);
-- 
2.14.3