This allows sharing flat views between address spaces when the same root memory region is used when creating a new address space.
This adds a global list of flat views and a list of attached address spaces per a flat view. Each address space references a flat view. This hard codes the dispatch tree building instead of doing so via a memory listener. Signed-off-by: Alexey Kardashevskiy <a...@ozlabs.ru> --- This was suggested by Paolo in https://lists.gnu.org/archive/html/qemu-devel/2017-08/msg05011.html I am not putting "Suggested-by" as I want to make sure that this is doing what was actually suggested. --- include/exec/memory-internal.h | 6 +- include/exec/memory.h | 9 +- exec.c | 58 ++-------- hw/pci/pci.c | 3 +- memory.c | 253 +++++++++++++++++++++++++++-------------- 5 files changed, 187 insertions(+), 142 deletions(-) diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index fb467acdba..8516e0b48f 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -22,9 +22,11 @@ #ifndef CONFIG_USER_ONLY typedef struct AddressSpaceDispatch AddressSpaceDispatch; -void address_space_init_dispatch(AddressSpace *as); void address_space_unregister(AddressSpace *as); -void address_space_destroy_dispatch(AddressSpace *as); +void address_space_dispatch_free(AddressSpaceDispatch *d); +AddressSpaceDispatch *mem_begin(void); +void mem_commit(AddressSpaceDispatch *d); +void mem_add(AddressSpaceDispatch *d, MemoryRegionSection *section); extern const MemoryRegionOps unassigned_mem_ops; diff --git a/include/exec/memory.h b/include/exec/memory.h index 83e82e90d5..41ab165302 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -27,6 +27,7 @@ #include "qemu/rcu.h" #include "hw/qdev-core.h" +typedef struct AddressSpaceDispatch AddressSpaceDispatch; #define RAM_ADDR_INVALID (~(ram_addr_t)0) #define MAX_PHYS_ADDR_SPACE_BITS 62 @@ -312,6 +313,7 @@ struct MemoryListener { }; AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as); +MemoryRegion *address_space_root(AddressSpace *as); /** * AddressSpace: describes a mapping of addresses to #MemoryRegion objects @@ -320,20 +322,17 @@ struct AddressSpace { /* All fields are private. */ struct rcu_head rcu; char *name; - MemoryRegion *root; - int ref_count; - bool malloced; /* Accessed via RCU. */ struct FlatView *current_map; int ioeventfd_nb; struct MemoryRegionIoeventfd *ioeventfds; - struct AddressSpaceDispatch *dispatch; - struct AddressSpaceDispatch *next_dispatch; + MemoryListener dispatch_listener; QTAILQ_HEAD(memory_listeners_as, MemoryListener) listeners; QTAILQ_ENTRY(AddressSpace) address_spaces_link; + QTAILQ_ENTRY(AddressSpace) flat_view_link; }; /** diff --git a/exec.c b/exec.c index 66f01f5fce..51243f57f4 100644 --- a/exec.c +++ b/exec.c @@ -188,15 +188,12 @@ typedef struct PhysPageMap { } PhysPageMap; struct AddressSpaceDispatch { - struct rcu_head rcu; - MemoryRegionSection *mru_section; /* This is a multi-level map on the physical address space. * The bottom level has pointers to MemoryRegionSections. */ PhysPageEntry phys_map; PhysPageMap map; - AddressSpace *as; }; typedef struct AddressSpaceDispatch AddressSpaceDispatch; @@ -240,11 +237,6 @@ struct DirtyBitmapSnapshot { unsigned long dirty[]; }; -AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as) -{ - return atomic_rcu_read(&as->dispatch); -} - #endif #if !defined(CONFIG_USER_ONLY) @@ -1354,10 +1346,8 @@ static void register_multipage(AddressSpaceDispatch *d, phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); } -static void mem_add(MemoryListener *listener, MemoryRegionSection *section) +void mem_add(AddressSpaceDispatch *d, MemoryRegionSection *section) { - AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener); - AddressSpaceDispatch *d = as->next_dispatch; MemoryRegionSection now = *section, remain = *section; Int128 page_size = int128_make64(TARGET_PAGE_SIZE); @@ -2680,9 +2670,8 @@ static void io_mem_init(void) NULL, UINT64_MAX); } -static void mem_begin(MemoryListener *listener) +AddressSpaceDispatch *mem_begin(void) { - AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener); AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); uint16_t n; @@ -2696,27 +2685,19 @@ static void mem_begin(MemoryListener *listener) assert(n == PHYS_SECTION_WATCH); d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 }; - as->next_dispatch = d; + + return d; } -static void address_space_dispatch_free(AddressSpaceDispatch *d) +void address_space_dispatch_free(AddressSpaceDispatch *d) { phys_sections_free(&d->map); g_free(d); } -static void mem_commit(MemoryListener *listener) +void mem_commit(AddressSpaceDispatch *d) { - AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener); - AddressSpaceDispatch *cur = as->dispatch; - AddressSpaceDispatch *next = as->next_dispatch; - - phys_page_compact_all(next, next->map.nodes_nb); - - atomic_rcu_set(&as->dispatch, next); - if (cur) { - call_rcu(cur, address_space_dispatch_free, rcu); - } + phys_page_compact_all(d, d->map.nodes_nb); } static void tcg_commit(MemoryListener *listener) @@ -2732,39 +2713,16 @@ static void tcg_commit(MemoryListener *listener) * We reload the dispatch pointer now because cpu_reloading_memory_map() * may have split the RCU critical section. */ - d = atomic_rcu_read(&cpuas->as->dispatch); + d = address_space_to_dispatch(cpuas->as); atomic_rcu_set(&cpuas->memory_dispatch, d); tlb_flush(cpuas->cpu); } -void address_space_init_dispatch(AddressSpace *as) -{ - as->dispatch = NULL; - as->dispatch_listener = (MemoryListener) { - .begin = mem_begin, - .commit = mem_commit, - .region_add = mem_add, - .region_nop = mem_add, - .priority = 0, - }; - memory_listener_register(&as->dispatch_listener, as); -} - void address_space_unregister(AddressSpace *as) { memory_listener_unregister(&as->dispatch_listener); } -void address_space_destroy_dispatch(AddressSpace *as) -{ - AddressSpaceDispatch *d = as->dispatch; - - atomic_rcu_set(&as->dispatch, NULL); - if (d) { - call_rcu(d, address_space_dispatch_free, rcu); - } -} - static void memory_map_init(void) { system_memory = g_malloc(sizeof(*system_memory)); diff --git a/hw/pci/pci.c b/hw/pci/pci.c index 258fbe51e2..86b9e419fd 100644 --- a/hw/pci/pci.c +++ b/hw/pci/pci.c @@ -88,7 +88,8 @@ static void pci_init_bus_master(PCIDevice *pci_dev) memory_region_init_alias(&pci_dev->bus_master_enable_region, OBJECT(pci_dev), "bus master", - dma_as->root, 0, memory_region_size(dma_as->root)); + address_space_root(dma_as), 0, + memory_region_size(address_space_root(dma_as))); memory_region_set_enabled(&pci_dev->bus_master_enable_region, false); memory_region_add_subregion(&pci_dev->bus_master_container_region, 0, &pci_dev->bus_master_enable_region); diff --git a/memory.c b/memory.c index c6904a7deb..385a507511 100644 --- a/memory.c +++ b/memory.c @@ -47,6 +47,9 @@ static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners static QTAILQ_HEAD(, AddressSpace) address_spaces = QTAILQ_HEAD_INITIALIZER(address_spaces); +static QTAILQ_HEAD(FlatViewList, FlatView) flat_views + = QTAILQ_HEAD_INITIALIZER(flat_views); + typedef struct AddrRange AddrRange; /* @@ -230,6 +233,11 @@ struct FlatView { FlatRange *ranges; unsigned nr; unsigned nr_allocated; + MemoryRegion *root; + struct AddressSpaceDispatch *dispatch; + + QTAILQ_ENTRY(FlatView) flat_views_link; + QTAILQ_HEAD(address_spaces, AddressSpace) address_spaces; }; typedef struct AddressSpaceOps AddressSpaceOps; @@ -259,12 +267,19 @@ static bool flatrange_equal(FlatRange *a, FlatRange *b) && a->readonly == b->readonly; } -static void flatview_init(FlatView *view) +static void flatview_ref(FlatView *view); + +static FlatView *flatview_alloc(MemoryRegion *mr_root) { + FlatView *view; + + view = g_new0(FlatView, 1); view->ref = 1; - view->ranges = NULL; - view->nr = 0; - view->nr_allocated = 0; + view->root = mr_root; + memory_region_ref(mr_root); + QTAILQ_INIT(&view->address_spaces); + + return view; } /* Insert a range into a given position. Caller is responsible for maintaining @@ -292,6 +307,10 @@ static void flatview_destroy(FlatView *view) memory_region_unref(view->ranges[i].mr); } g_free(view->ranges); + if (view->dispatch) { + address_space_dispatch_free(view->dispatch); + } + memory_region_unref(view->root); g_free(view); } @@ -303,7 +322,7 @@ static void flatview_ref(FlatView *view) static void flatview_unref(FlatView *view) { if (atomic_fetch_dec(&view->ref) == 1) { - flatview_destroy(view); + call_rcu(view, flatview_destroy, rcu); } } @@ -608,7 +627,7 @@ static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) mr = mr->container; } QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - if (mr == as->root) { + if (mr == as->current_map->root) { return as; } } @@ -702,23 +721,6 @@ static void render_memory_region(FlatView *view, } } -/* Render a memory topology into a list of disjoint absolute ranges. */ -static FlatView *generate_memory_topology(MemoryRegion *mr) -{ - FlatView *view; - - view = g_new(FlatView, 1); - flatview_init(view); - - if (mr) { - render_memory_region(view, mr, int128_zero(), - addrrange_make(int128_zero(), int128_2_64()), false); - } - flatview_simplify(view); - - return view; -} - static void address_space_add_del_ioeventfds(AddressSpace *as, MemoryRegionIoeventfd *fds_new, unsigned fds_new_nb, @@ -769,12 +771,10 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, } } -static FlatView *address_space_get_flatview(AddressSpace *as) +static FlatView *flatview_get(FlatView *view) { - FlatView *view; - rcu_read_lock(); - view = atomic_rcu_read(&as->current_map); + view = atomic_rcu_read(&view); flatview_ref(view); rcu_read_unlock(); return view; @@ -789,7 +789,7 @@ static void address_space_update_ioeventfds(AddressSpace *as) AddrRange tmp; unsigned i; - view = address_space_get_flatview(as); + view = flatview_get(as->current_map); FOR_EACH_FLAT_RANGE(fr, view) { for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, @@ -881,28 +881,89 @@ static void address_space_update_topology_pass(AddressSpace *as, } } - -static void address_space_update_topology(AddressSpace *as) +static FlatView *address_space_update_flatview(FlatView *view) { - FlatView *old_view = address_space_get_flatview(as); - FlatView *new_view = generate_memory_topology(as->root); + AddressSpace *as, *asnext; + FlatView *old_view = flatview_get(view); + MemoryRegion *root = old_view->root; + FlatView *new_view = flatview_alloc(root); + unsigned iold, inew; + FlatRange *frold, *frnew; - address_space_update_topology_pass(as, old_view, new_view, false); - address_space_update_topology_pass(as, old_view, new_view, true); + if (root) { + render_memory_region(new_view, root, int128_zero(), + addrrange_make(int128_zero(), int128_2_64()), + false); + flatview_simplify(new_view); + } - /* Writes are protected by the BQL. */ - atomic_rcu_set(&as->current_map, new_view); - call_rcu(old_view, flatview_unref, rcu); + new_view->dispatch = mem_begin(); - /* Note that all the old MemoryRegions are still alive up to this - * point. This relieves most MemoryListeners from the need to - * ref/unref the MemoryRegions they get---unless they use them - * outside the iothread mutex, in which case precise reference - * counting is necessary. + /* + * FIXME: this is cut-n-paste from address_space_update_topology_pass, + * simplify it */ + iold = inew = 0; + while (iold < old_view->nr || inew < new_view->nr) { + if (iold < old_view->nr) { + frold = &old_view->ranges[iold]; + } else { + frold = NULL; + } + if (inew < new_view->nr) { + frnew = &new_view->ranges[inew]; + } else { + frnew = NULL; + } + + if (frold + && (!frnew + || int128_lt(frold->addr.start, frnew->addr.start) + || (int128_eq(frold->addr.start, frnew->addr.start) + && !flatrange_equal(frold, frnew)))) { + ++iold; + } else if (frold && frnew && flatrange_equal(frold, frnew)) { + /* In both and unchanged (except logging may have changed) */ + MemoryRegionSection mrs = section_from_flat_range(frnew, + new_view->dispatch); + + mem_add(new_view->dispatch, &mrs); + + ++iold; + ++inew; + } else { + /* In new */ + MemoryRegionSection mrs = section_from_flat_range(frnew, + new_view->dispatch); + + mem_add(new_view->dispatch, &mrs); + + ++inew; + } + } + + mem_commit(new_view->dispatch); + + QTAILQ_FOREACH(as, &old_view->address_spaces, flat_view_link) { + address_space_update_topology_pass(as, old_view, new_view, false); + address_space_update_topology_pass(as, old_view, new_view, true); + } + + QTAILQ_FOREACH_SAFE(as, &old_view->address_spaces, flat_view_link, asnext) { + QTAILQ_REMOVE(&old_view->address_spaces, as, flat_view_link); + flatview_unref(old_view); + + atomic_rcu_set(&as->current_map, new_view); + + flatview_ref(new_view); + QTAILQ_INSERT_TAIL(&new_view->address_spaces, as, flat_view_link); + + address_space_update_ioeventfds(as); + } + flatview_unref(old_view); - address_space_update_ioeventfds(as); + return new_view; } void memory_region_transaction_begin(void) @@ -921,11 +982,31 @@ void memory_region_transaction_commit(void) --memory_region_transaction_depth; if (!memory_region_transaction_depth) { if (memory_region_update_pending) { + FlatView *view, *vnext; + struct FlatViewList fwstmp = QTAILQ_HEAD_INITIALIZER(fwstmp); + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); - QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - address_space_update_topology(as); + QTAILQ_FOREACH_SAFE(view, &flat_views, flat_views_link, vnext) { + FlatView *old_view, *new_view; + + old_view = flatview_get(view); + new_view = address_space_update_flatview(old_view); + + QTAILQ_REMOVE(&flat_views, old_view, flat_views_link); + flatview_unref(old_view); + flatview_unref(old_view); + + QTAILQ_INSERT_HEAD(&fwstmp, new_view, flat_views_link); + + flatview_unref(new_view); } + + QTAILQ_FOREACH_SAFE(view, &fwstmp, flat_views_link, vnext) { + QTAILQ_REMOVE(&fwstmp, view, flat_views_link); + QTAILQ_INSERT_HEAD(&flat_views, view, flat_views_link); + } + memory_region_update_pending = false; MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); } else if (ioeventfd_update_pending) { @@ -1834,7 +1915,7 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) continue; } as = listener->address_space; - view = address_space_get_flatview(as); + view = flatview_get(as->current_map); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->mr == mr) { MemoryRegionSection mrs = section_from_flat_range(fr, @@ -1938,7 +2019,7 @@ static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpa AddrRange tmp; MemoryRegionSection section; - view = address_space_get_flatview(as); + view = flatview_get(as->current_map); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->mr == mr) { section = (MemoryRegionSection) { @@ -2350,7 +2431,7 @@ void memory_global_dirty_log_sync(void) continue; } as = listener->address_space; - view = address_space_get_flatview(as); + view = flatview_get(as->current_map); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->dirty_log_mask) { MemoryRegionSection mrs = section_from_flat_range(fr, @@ -2436,7 +2517,7 @@ static void listener_add_address_space(MemoryListener *listener, } } - view = address_space_get_flatview(as); + view = flatview_get(as->current_map); FOR_EACH_FLAT_RANGE(fr, view) { MemoryRegionSection section = { .mr = fr->mr, @@ -2615,67 +2696,72 @@ void memory_region_invalidate_mmio_ptr(MemoryRegion *mr, hwaddr offset, void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) { - memory_region_ref(root); - memory_region_transaction_begin(); - as->ref_count = 1; - as->root = root; - as->malloced = false; - as->current_map = g_new(FlatView, 1); - flatview_init(as->current_map); + FlatView *view; + + as->current_map = NULL; as->ioeventfd_nb = 0; as->ioeventfds = NULL; QTAILQ_INIT(&as->listeners); QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); as->name = g_strdup(name ? name : "anonymous"); - address_space_init_dispatch(as); - memory_region_update_pending |= root->enabled; - memory_region_transaction_commit(); + + QTAILQ_FOREACH(view, &flat_views, flat_views_link) { + assert(root); + if (view->root == root) { + as->current_map = flatview_get(view); + break; + } + } + + if (!as->current_map) { + as->current_map = flatview_alloc(root); + + QTAILQ_INSERT_TAIL(&flat_views, as->current_map, flat_views_link); + } + + QTAILQ_INSERT_TAIL(&as->current_map->address_spaces, as, flat_view_link); +} + +MemoryRegion *address_space_root(AddressSpace *as) +{ + return as->current_map->root; +} + +AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as) +{ + return atomic_rcu_read(&as->current_map)->dispatch; } static void do_address_space_destroy(AddressSpace *as) { - bool do_free = as->malloced; + FlatView *view = flatview_get(as->current_map); - address_space_destroy_dispatch(as); assert(QTAILQ_EMPTY(&as->listeners)); - flatview_unref(as->current_map); + QTAILQ_REMOVE(&view->address_spaces, as, flat_view_link); + flatview_unref(view); + + flatview_unref(view); + g_free(as->name); g_free(as->ioeventfds); - memory_region_unref(as->root); - if (do_free) { - g_free(as); - } + g_free(as); } AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name) { AddressSpace *as; - QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - if (root == as->root && as->malloced) { - as->ref_count++; - return as; - } - } - as = g_malloc0(sizeof *as); address_space_init(as, root, name); - as->malloced = true; + return as; } void address_space_destroy(AddressSpace *as) { - MemoryRegion *root = as->root; - - as->ref_count--; - if (as->ref_count) { - return; - } /* Flush out anything from MemoryListeners listening in on this */ memory_region_transaction_begin(); - as->root = NULL; memory_region_transaction_commit(); QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); address_space_unregister(as); @@ -2684,7 +2770,6 @@ void address_space_destroy(AddressSpace *as) * entries that the guest should never use. Wait for the old * values to expire before freeing the data. */ - as->root = root; call_rcu(as, do_address_space_destroy, rcu); } @@ -2816,7 +2901,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, static void mtree_print_flatview(fprintf_function p, void *f, AddressSpace *as) { - FlatView *view = address_space_get_flatview(as); + FlatView *view = flatview_get(as->current_map); FlatRange *range = &view->ranges[0]; MemoryRegion *mr; int n = view->nr; @@ -2873,7 +2958,7 @@ void mtree_info(fprintf_function mon_printf, void *f, bool flatview) QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { mon_printf(f, "address-space: %s\n", as->name); - mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head); + mtree_print_mr(mon_printf, f, as->current_map->root, 1, 0, &ml_head); mon_printf(f, "\n"); } -- 2.11.0
