Now that release_pages is scaling better with concurrent removals from
the LRU, the performance results (included below) showed increased
contention on lru_lock in the add-to-LRU path.
To alleviate some of this contention, do more work outside the LRU lock.
Prepare a local list of pages to be spliced onto the front of the LRU,
including setting PageLRU in each page, before taking lru_lock. Since
other threads use this page flag in certain checks outside lru_lock,
ensure each page's LRU links have been properly initialized before
setting the flag, and use memory barriers accordingly.
Performance Results
This is a will-it-scale run of page_fault1 using 4 different kernels.
kernel kern #
4.15-rc2 1
large-zone-batch 2
lru-lock-base 3
lru-lock-splice 4
Each kernel builds on the last. The first is a baseline, the second
makes zone->lock more scalable by increasing an order-0 per-cpu
pagelist's 'batch' and 'high' values to 310 and 1860 respectively
(courtesy of Aaron Lu's patch), the third scales lru_lock without
splicing pages (the previous patch in this series), and the fourth adds
page splicing (this patch).
N tasks mmap, fault, and munmap anonymous pages in a loop until the test
time has elapsed.
The process case generally does better than the thread case most likely
because of mmap_sem acting as a bottleneck. There's ongoing work
upstream[*] to scale this lock, however, and once it goes in, my
hypothesis is the thread numbers here will improve.
kern # ntask proc thr proc stdev thr stdev
speedup speedup pgf/s pgf/s
1 1 705,533 1,644 705,227 1,122
2 1 2.5% 2.8% 722,912 453 724,807 728
3 1 2.6% 2.6% 724,215 653 723,213 941
4 1 2.3% 2.8% 721,746 272 724,944 728
kern # ntask proc thr proc stdev thr stdev
speedup speedup pgf/s pgf/s
1 4 2,525,487 7,428 1,973,616 12,568
2 4 2.6% 7.6% 2,590,699 6,968 2,123,570 10,350
3 4 2.3% 4.4% 2,584,668 12,833 2,059,822 10,748
4 4 4.7% 5.2% 2,643,251 13,297 2,076,808 9,506
kern # ntask proc thr proc stdev thr stdev
speedup speedup pgf/s pgf/s
1 16 6,444,656 20,528 3,226,356 32,874
2 16 1.9% 10.4% 6,566,846 20,803 3,560,437 64,019
3 16 18.3% 6.8% 7,624,749 58,497 3,447,109 67,734
4 16 28.2% 2.5% 8,264,125 31,677 3,306,679 69,443
kern # ntask proc thr proc stdev thr stdev
speedup speedup pgf/s pgf/s
1 32 11,564,988 32,211 2,456,507 38,898
2 32 1.8% 1.5% 11,777,119 45,418 2,494,064 27,964
3 32 16.1% -2.7% 13,426,746 94,057 2,389,934 40,186
4 32 26.2% 1.2% 14,593,745 28,121 2,486,059 42,004
kern # ntask proc thr proc stdev thr stdev
speedup speedup pgf/s pgf/s
1 64 12,080,629 33,676 2,443,043 61,973
2 64 3.9% 9.9% 12,551,136 206,202 2,684,632 69,483
3 64 15.0% -3.8% 13,892,933 351,657 2,351,232 67,875
4 64 21.9% 1.8% 14,728,765 64,945 2,485,940 66,839
[*] https://lwn.net/Articles/724502/ Range reader/writer locks
https://lwn.net/Articles/744188/ Speculative page faults
Signed-off-by: Daniel Jordan <daniel.m.jor...@oracle.com>
---
mm/memcontrol.c | 1 +
mm/mlock.c | 1 +
mm/swap.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--
mm/vmscan.c | 1 +
4 files changed, 112 insertions(+), 4 deletions(-)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 99a54df760e3..6911626f29b2 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2077,6 +2077,7 @@ static void lock_page_lru(struct page *page, int
*isolated)
lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
ClearPageLRU(page);
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec, page_lru(page));
*isolated = 1;
} else
diff --git a/mm/mlock.c b/mm/mlock.c
index 6ba6a5887aeb..da294c5bbc2c 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -109,6 +109,7 @@ static bool __munlock_isolate_lru_page(struct page *page,
bool getpage)
if (getpage)
get_page(page);
ClearPageLRU(page);
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec, page_lru(page));
return true;
}
diff --git a/mm/swap.c b/mm/swap.c
index a302224293ad..46a98dc8e9ad 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -220,6 +220,7 @@ static void pagevec_move_tail_fn(struct page *page, struct
lruvec *lruvec,
int *pgmoved = arg;
if (PageLRU(page) && !PageUnevictable(page)) {
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec, page_lru(page));
ClearPageActive(page);
add_page_to_lru_list_tail(page, lruvec, page_lru(page));
@@ -277,6 +278,7 @@ static void __activate_page(struct page *page, struct
lruvec *lruvec,
int file = page_is_file_cache(page);
int lru = page_lru_base_type(page);
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec, lru);
SetPageActive(page);
lru += LRU_ACTIVE;
@@ -544,6 +546,7 @@ static void lru_deactivate_file_fn(struct page *page,
struct lruvec *lruvec,
file = page_is_file_cache(page);
lru = page_lru_base_type(page);
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec, lru + active);
ClearPageActive(page);
ClearPageReferenced(page);
@@ -578,6 +581,7 @@ static void lru_lazyfree_fn(struct page *page, struct
lruvec *lruvec,
!PageSwapCache(page) && !PageUnevictable(page)) {
bool active = PageActive(page);
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec,
LRU_INACTIVE_ANON + active);
ClearPageActive(page);
@@ -903,6 +907,60 @@ static void __pagevec_lru_add_fn(struct page *page, struct
lruvec *lruvec,
trace_mm_lru_insertion(page, lru);
}
+#define MAX_LRU_SPLICES 4
+
+struct lru_splice {
+ struct list_head list;
+ struct lruvec *lruvec;
+ enum lru_list lru;
+ int nid;
+ int zid;
+ size_t nr_pages;
+};
+
+/*
+ * Adds a page to a local list for splicing, or else to the singletons
+ * list for individual processing.
+ *
+ * Returns the new number of splices in the splices list.
+ */
+size_t add_page_to_lru_splice(struct lru_splice *splices, size_t nr_splices,
+ struct list_head *singletons, struct page *page)
+{
+ int i;
+ enum lru_list lru = page_lru(page);
+ enum zone_type zid = page_zonenum(page);
+ int nid = page_to_nid(page);
+ struct lruvec *lruvec;
+
+ VM_BUG_ON_PAGE(PageLRU(page), page);
+
+ lruvec = mem_cgroup_page_lruvec(page, NODE_DATA(nid));
+
+ for (i = 0; i < nr_splices; ++i) {
+ if (splices[i].lruvec == lruvec && splices[i].zid == zid) {
+ list_add(&page->lru, &splices[i].list);
+ splices[nr_splices].nr_pages += hpage_nr_pages(page);
+ return nr_splices;
+ }
+ }
+
+ if (nr_splices < MAX_LRU_SPLICES) {
+ INIT_LIST_HEAD(&splices[nr_splices].list);
+ splices[nr_splices].lruvec = lruvec;
+ splices[nr_splices].lru = lru;
+ splices[nr_splices].nid = nid;
+ splices[nr_splices].zid = zid;
+ splices[nr_splices].nr_pages = hpage_nr_pages(page);
+ list_add(&page->lru, &splices[nr_splices].list);
+ ++nr_splices;
+ } else {
+ list_add(&page->lru, singletons);
+ }
+
+ return nr_splices;
+}
+
/*
* Add the passed pages to the LRU, then drop the caller's refcount
* on them. Reinitialises the caller's pagevec.
@@ -911,12 +969,59 @@ void __pagevec_lru_add(struct pagevec *pvec)
{
int i;
struct pglist_data *pgdat = NULL;
- struct lruvec *lruvec;
unsigned long flags = 0;
+ struct lru_splice splices[MAX_LRU_SPLICES];
+ size_t nr_splices = 0;
+ LIST_HEAD(singletons);
+ struct page *page, *next;
- for (i = 0; i < pagevec_count(pvec); i++) {
- struct page *page = pvec->pages[i];
- struct pglist_data *pagepgdat = page_pgdat(page);
+ /*
+ * Sort the pages into local lists to splice onto the LRU once we
+ * hold lru_lock. In the common case there should be few of these
+ * local lists.
+ */
+ for (i = 0; i < pagevec_count(pvec); ++i) {
+ page = pvec->pages[i];
+ nr_splices = add_page_to_lru_splice(splices, nr_splices,
+ &singletons, page);
+ }
+
+ /*
+ * Paired with read barriers where we check PageLRU and modify
+ * page->lru, for example pagevec_move_tail_fn.
+ */
+ smp_wmb();
+
+ for (i = 0; i < pagevec_count(pvec); i++)
+ SetPageLRU(pvec->pages[i]);
+
+ for (i = 0; i < nr_splices; ++i) {
+ struct lru_splice *s = &splices[i];
+ struct pglist_data *splice_pgdat = NODE_DATA(s->nid);
+
+ if (splice_pgdat != pgdat) {
+ if (pgdat)
+ spin_unlock_irqrestore(&pgdat->lru_lock, flags);
+ pgdat = splice_pgdat;
+ spin_lock_irqsave(&pgdat->lru_lock, flags);
+ }
+
+ update_lru_size(s->lruvec, s->lru, s->zid, s->nr_pages);
+ list_splice(&s->list, lru_head(&s->lruvec->lists[s->lru]));
+ update_page_reclaim_stat(s->lruvec, is_file_lru(s->lru),
+ is_active_lru(s->lru));
+ /* XXX add splice tracepoint */
+ }
+
+ while (!list_empty(&singletons)) {
+ struct pglist_data *pagepgdat;
+ struct lruvec *lruvec;
+ struct list_head *list;
+
+ list = singletons.next;
+ page = list_entry(list, struct page, lru);
+ list_del(list);
+ pagepgdat = page_pgdat(page);
if (pagepgdat != pgdat) {
if (pgdat)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 7f5ff0bb133f..338850ad03a6 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1629,6 +1629,7 @@ int isolate_lru_page(struct page *page)
int lru = page_lru(page);
get_page(page);
ClearPageLRU(page);
+ smp_rmb(); /* Pairs with smp_wmb in __pagevec_lru_add */
del_page_from_lru_list(page, lruvec, lru);
ret = 0;
}
--
2.16.1
