Re: [PATCH v3 13/20] cputlb: Move NOTDIRTY handling from I/O path to TLB path
On 9/22/19 5:54 AM, Richard Henderson wrote:
> Pages that we want to track for NOTDIRTY are RAM. We do not
> really need to go through the I/O path to handle them.
>
> Signed-off-by: Richard Henderson
> ---
> include/exec/cpu-common.h | 2 --
> accel/tcg/cputlb.c| 26 +---
> exec.c| 50 ---
> memory.c | 16 -
> 4 files changed, 23 insertions(+), 71 deletions(-)
>
> diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
> index 1c0e03ddc2..81753bbb34 100644
> --- a/include/exec/cpu-common.h
> +++ b/include/exec/cpu-common.h
> @@ -100,8 +100,6 @@ void qemu_flush_coalesced_mmio_buffer(void);
>
> void cpu_flush_icache_range(hwaddr start, hwaddr len);
>
> -extern struct MemoryRegion io_mem_notdirty;
> -
> typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);
>
> int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);
> diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
> index 7ab523d7ec..b7bd738115 100644
> --- a/accel/tcg/cputlb.c
> +++ b/accel/tcg/cputlb.c
> @@ -904,7 +904,7 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry
> *iotlbentry,
> mr = section->mr;
> mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
> cpu->mem_io_pc = retaddr;
> -if (mr != _mem_notdirty && !cpu->can_do_io) {
> +if (!cpu->can_do_io) {
> cpu_io_recompile(cpu, retaddr);
> }
>
> @@ -945,7 +945,7 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry
> *iotlbentry,
> section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);
> mr = section->mr;
> mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
> -if (mr != _mem_notdirty && !cpu->can_do_io) {
> +if (!cpu->can_do_io) {
> cpu_io_recompile(cpu, retaddr);
> }
> cpu->mem_io_vaddr = addr;
> @@ -1606,7 +1606,7 @@ store_helper(CPUArchState *env, target_ulong addr,
> uint64_t val,
> }
>
> /* Handle I/O access. */
> -if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {
> +if (tlb_addr & TLB_MMIO) {
> io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,
>op ^ (tlb_addr & TLB_BSWAP ? MO_BSWAP : 0));
> return;
> @@ -1619,6 +1619,26 @@ store_helper(CPUArchState *env, target_ulong addr,
> uint64_t val,
>
> haddr = (void *)((uintptr_t)addr + entry->addend);
>
> +/* Handle clean RAM pages. */
> +if (tlb_addr & TLB_NOTDIRTY) {
> +NotDirtyInfo ndi;
> +
> +/* We require mem_io_pc in tb_invalidate_phys_page_range. */
> +env_cpu(env)->mem_io_pc = retaddr;
> +
> +memory_notdirty_write_prepare(, env_cpu(env), addr,
> + addr + iotlbentry->addr, size);
> +
> +if (unlikely(tlb_addr & TLB_BSWAP)) {
> +direct_swap(haddr, val);
> +} else {
> +direct(haddr, val);
> +}
> +
> +memory_notdirty_write_complete();
> +return;
> +}
> +
> if (unlikely(tlb_addr & TLB_BSWAP)) {
> direct_swap(haddr, val);
> } else {
> diff --git a/exec.c b/exec.c
> index e21e068535..abf58b68a0 100644
> --- a/exec.c
> +++ b/exec.c
> @@ -88,7 +88,6 @@ static MemoryRegion *system_io;
> AddressSpace address_space_io;
> AddressSpace address_space_memory;
>
> -MemoryRegion io_mem_notdirty;
> static MemoryRegion io_mem_unassigned;
> #endif
>
> @@ -157,7 +156,6 @@ typedef struct subpage_t {
> } subpage_t;
>
> #define PHYS_SECTION_UNASSIGNED 0
> -#define PHYS_SECTION_NOTDIRTY 1
>
> static void io_mem_init(void);
> static void memory_map_init(void);
> @@ -1438,9 +1436,6 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,
> if (memory_region_is_ram(section->mr)) {
> /* Normal RAM. */
> iotlb = memory_region_get_ram_addr(section->mr) + xlat;
> -if (!section->readonly) {
> -iotlb |= PHYS_SECTION_NOTDIRTY;
> -}
> } else {
> AddressSpaceDispatch *d;
>
> @@ -2749,42 +2744,6 @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)
> }
> }
>
> -/* Called within RCU critical section. */
> -static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
> - uint64_t val, unsigned size)
> -{
> -NotDirtyInfo ndi;
> -
> -memory_notdirty_write_prepare(, current_cpu,
> current_cpu->mem_io_vaddr,
> - ram_addr, size);
> -
> -stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
> -memory_notdirty_write_complete();
> -}
> -
> -static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
> - unsigned size, bool is_write,
> - MemTxAttrs attrs)
> -{
> -return is_write;
> -}
> -
> -static const MemoryRegionOps notdirty_mem_ops = {
> -
Re: [PATCH v3 13/20] cputlb: Move NOTDIRTY handling from I/O path to TLB path
On 22.09.19 05:54, Richard Henderson wrote:
> Pages that we want to track for NOTDIRTY are RAM. We do not
> really need to go through the I/O path to handle them.
>
> Signed-off-by: Richard Henderson
> ---
> include/exec/cpu-common.h | 2 --
> accel/tcg/cputlb.c| 26 +---
> exec.c| 50 ---
> memory.c | 16 -
> 4 files changed, 23 insertions(+), 71 deletions(-)
>
> diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
> index 1c0e03ddc2..81753bbb34 100644
> --- a/include/exec/cpu-common.h
> +++ b/include/exec/cpu-common.h
> @@ -100,8 +100,6 @@ void qemu_flush_coalesced_mmio_buffer(void);
>
> void cpu_flush_icache_range(hwaddr start, hwaddr len);
>
> -extern struct MemoryRegion io_mem_notdirty;
> -
> typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);
>
> int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);
> diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
> index 7ab523d7ec..b7bd738115 100644
> --- a/accel/tcg/cputlb.c
> +++ b/accel/tcg/cputlb.c
> @@ -904,7 +904,7 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry
> *iotlbentry,
> mr = section->mr;
> mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
> cpu->mem_io_pc = retaddr;
> -if (mr != _mem_notdirty && !cpu->can_do_io) {
> +if (!cpu->can_do_io) {
> cpu_io_recompile(cpu, retaddr);
> }
>
> @@ -945,7 +945,7 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry
> *iotlbentry,
> section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);
> mr = section->mr;
> mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
> -if (mr != _mem_notdirty && !cpu->can_do_io) {
> +if (!cpu->can_do_io) {
> cpu_io_recompile(cpu, retaddr);
> }
> cpu->mem_io_vaddr = addr;
> @@ -1606,7 +1606,7 @@ store_helper(CPUArchState *env, target_ulong addr,
> uint64_t val,
> }
>
> /* Handle I/O access. */
> -if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {
> +if (tlb_addr & TLB_MMIO) {
> io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,
>op ^ (tlb_addr & TLB_BSWAP ? MO_BSWAP : 0));
> return;
> @@ -1619,6 +1619,26 @@ store_helper(CPUArchState *env, target_ulong addr,
> uint64_t val,
>
> haddr = (void *)((uintptr_t)addr + entry->addend);
>
> +/* Handle clean RAM pages. */
> +if (tlb_addr & TLB_NOTDIRTY) {
> +NotDirtyInfo ndi;
> +
> +/* We require mem_io_pc in tb_invalidate_phys_page_range. */
> +env_cpu(env)->mem_io_pc = retaddr;
> +
> +memory_notdirty_write_prepare(, env_cpu(env), addr,
> + addr + iotlbentry->addr, size);
> +
> +if (unlikely(tlb_addr & TLB_BSWAP)) {
> +direct_swap(haddr, val);
> +} else {
> +direct(haddr, val);
> +}
> +
> +memory_notdirty_write_complete();
> +return;
> +}
> +
> if (unlikely(tlb_addr & TLB_BSWAP)) {
> direct_swap(haddr, val);
> } else {
> diff --git a/exec.c b/exec.c
> index e21e068535..abf58b68a0 100644
> --- a/exec.c
> +++ b/exec.c
> @@ -88,7 +88,6 @@ static MemoryRegion *system_io;
> AddressSpace address_space_io;
> AddressSpace address_space_memory;
>
> -MemoryRegion io_mem_notdirty;
> static MemoryRegion io_mem_unassigned;
> #endif
>
> @@ -157,7 +156,6 @@ typedef struct subpage_t {
> } subpage_t;
>
> #define PHYS_SECTION_UNASSIGNED 0
> -#define PHYS_SECTION_NOTDIRTY 1
>
> static void io_mem_init(void);
> static void memory_map_init(void);
> @@ -1438,9 +1436,6 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,
> if (memory_region_is_ram(section->mr)) {
> /* Normal RAM. */
> iotlb = memory_region_get_ram_addr(section->mr) + xlat;
> -if (!section->readonly) {
> -iotlb |= PHYS_SECTION_NOTDIRTY;
> -}
> } else {
> AddressSpaceDispatch *d;
>
> @@ -2749,42 +2744,6 @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)
> }
> }
>
> -/* Called within RCU critical section. */
> -static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
> - uint64_t val, unsigned size)
> -{
> -NotDirtyInfo ndi;
> -
> -memory_notdirty_write_prepare(, current_cpu,
> current_cpu->mem_io_vaddr,
> - ram_addr, size);
> -
> -stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
> -memory_notdirty_write_complete();
> -}
> -
> -static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
> - unsigned size, bool is_write,
> - MemTxAttrs attrs)
> -{
> -return is_write;
> -}
> -
> -static const MemoryRegionOps notdirty_mem_ops = {
> -
[PATCH v3 13/20] cputlb: Move NOTDIRTY handling from I/O path to TLB path
Pages that we want to track for NOTDIRTY are RAM. We do not
really need to go through the I/O path to handle them.
Signed-off-by: Richard Henderson
---
include/exec/cpu-common.h | 2 --
accel/tcg/cputlb.c| 26 +---
exec.c| 50 ---
memory.c | 16 -
4 files changed, 23 insertions(+), 71 deletions(-)
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index 1c0e03ddc2..81753bbb34 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -100,8 +100,6 @@ void qemu_flush_coalesced_mmio_buffer(void);
void cpu_flush_icache_range(hwaddr start, hwaddr len);
-extern struct MemoryRegion io_mem_notdirty;
-
typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);
int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 7ab523d7ec..b7bd738115 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -904,7 +904,7 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry
*iotlbentry,
mr = section->mr;
mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
cpu->mem_io_pc = retaddr;
-if (mr != _mem_notdirty && !cpu->can_do_io) {
+if (!cpu->can_do_io) {
cpu_io_recompile(cpu, retaddr);
}
@@ -945,7 +945,7 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry
*iotlbentry,
section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);
mr = section->mr;
mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
-if (mr != _mem_notdirty && !cpu->can_do_io) {
+if (!cpu->can_do_io) {
cpu_io_recompile(cpu, retaddr);
}
cpu->mem_io_vaddr = addr;
@@ -1606,7 +1606,7 @@ store_helper(CPUArchState *env, target_ulong addr,
uint64_t val,
}
/* Handle I/O access. */
-if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {
+if (tlb_addr & TLB_MMIO) {
io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,
op ^ (tlb_addr & TLB_BSWAP ? MO_BSWAP : 0));
return;
@@ -1619,6 +1619,26 @@ store_helper(CPUArchState *env, target_ulong addr,
uint64_t val,
haddr = (void *)((uintptr_t)addr + entry->addend);
+/* Handle clean RAM pages. */
+if (tlb_addr & TLB_NOTDIRTY) {
+NotDirtyInfo ndi;
+
+/* We require mem_io_pc in tb_invalidate_phys_page_range. */
+env_cpu(env)->mem_io_pc = retaddr;
+
+memory_notdirty_write_prepare(, env_cpu(env), addr,
+ addr + iotlbentry->addr, size);
+
+if (unlikely(tlb_addr & TLB_BSWAP)) {
+direct_swap(haddr, val);
+} else {
+direct(haddr, val);
+}
+
+memory_notdirty_write_complete();
+return;
+}
+
if (unlikely(tlb_addr & TLB_BSWAP)) {
direct_swap(haddr, val);
} else {
diff --git a/exec.c b/exec.c
index e21e068535..abf58b68a0 100644
--- a/exec.c
+++ b/exec.c
@@ -88,7 +88,6 @@ static MemoryRegion *system_io;
AddressSpace address_space_io;
AddressSpace address_space_memory;
-MemoryRegion io_mem_notdirty;
static MemoryRegion io_mem_unassigned;
#endif
@@ -157,7 +156,6 @@ typedef struct subpage_t {
} subpage_t;
#define PHYS_SECTION_UNASSIGNED 0
-#define PHYS_SECTION_NOTDIRTY 1
static void io_mem_init(void);
static void memory_map_init(void);
@@ -1438,9 +1436,6 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,
if (memory_region_is_ram(section->mr)) {
/* Normal RAM. */
iotlb = memory_region_get_ram_addr(section->mr) + xlat;
-if (!section->readonly) {
-iotlb |= PHYS_SECTION_NOTDIRTY;
-}
} else {
AddressSpaceDispatch *d;
@@ -2749,42 +2744,6 @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)
}
}
-/* Called within RCU critical section. */
-static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
- uint64_t val, unsigned size)
-{
-NotDirtyInfo ndi;
-
-memory_notdirty_write_prepare(, current_cpu, current_cpu->mem_io_vaddr,
- ram_addr, size);
-
-stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
-memory_notdirty_write_complete();
-}
-
-static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
- unsigned size, bool is_write,
- MemTxAttrs attrs)
-{
-return is_write;
-}
-
-static const MemoryRegionOps notdirty_mem_ops = {
-.write = notdirty_mem_write,
-.valid.accepts = notdirty_mem_accepts,
-.endianness = DEVICE_NATIVE_ENDIAN,
-.valid = {
-.min_access_size = 1,
-.max_access_size = 8,
-.unaligned = false,
-},
-.impl = {
-.min_access_size = 1,
-.max_access_size = 8,
-.unaligned =
