try_to_compact_pages() defers a zone only after sync compaction fails, so a failed async direct compaction is never deferred and repeats the same futile zone scan on the next attempt. Deferring async compaction needs state of its own: sync compaction may still succeed on the pageblocks async skips, so it mustn't be deferred along with it.
Turn compact_considered, compact_defer_shift and compact_order_failed into arrays indexed by sync, like compact_cached_migrate_pfn, and pass the mode to the deferral helpers and tracepoints. All callers use the sync state for now and a reset clears both. The deferral tracepoints gain a sync field telling which state they report. Signed-off-by: Qiliang Yuan <[email protected]> --- include/linux/mmzone.h | 7 +++-- include/trace/events/compaction.h | 27 ++++++++++-------- mm/compaction.c | 60 +++++++++++++++++++++------------------ 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 94f9c3ff54160..91fbaa7aec6f9 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1140,10 +1140,11 @@ struct zone { * are skipped before trying again. The number attempted since * last failure is tracked with compact_considered. * compact_order_failed is the minimum compaction failed order. + * Indexed by sync, like compact_cached_migrate_pfn. */ - unsigned int compact_considered; - unsigned int compact_defer_shift; - int compact_order_failed; + unsigned int compact_considered[ASYNC_AND_SYNC]; + unsigned int compact_defer_shift[ASYNC_AND_SYNC]; + int compact_order_failed[ASYNC_AND_SYNC]; #endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA diff --git a/include/trace/events/compaction.h b/include/trace/events/compaction.h index d05759d185389..23ac6e1750623 100644 --- a/include/trace/events/compaction.h +++ b/include/trace/events/compaction.h @@ -238,14 +238,15 @@ DEFINE_EVENT(mm_compaction_suitable_template, mm_compaction_suitable, DECLARE_EVENT_CLASS(mm_compaction_defer_template, - TP_PROTO(struct zone *zone, int order), + TP_PROTO(struct zone *zone, int order, bool sync), - TP_ARGS(zone, order), + TP_ARGS(zone, order, sync), TP_STRUCT__entry( __field(int, nid) __field(enum zone_type, idx) __field(int, order) + __field(bool, sync) __field(unsigned int, considered) __field(unsigned int, defer_shift) __field(int, order_failed) @@ -255,15 +256,17 @@ DECLARE_EVENT_CLASS(mm_compaction_defer_template, __entry->nid = zone_to_nid(zone); __entry->idx = zone_idx(zone); __entry->order = order; - __entry->considered = zone->compact_considered; - __entry->defer_shift = zone->compact_defer_shift; - __entry->order_failed = zone->compact_order_failed; + __entry->sync = sync; + __entry->considered = zone->compact_considered[sync]; + __entry->defer_shift = zone->compact_defer_shift[sync]; + __entry->order_failed = zone->compact_order_failed[sync]; ), - TP_printk("node=%d zone=%-8s order=%d order_failed=%d consider=%u limit=%lu", + TP_printk("node=%d zone=%-8s order=%d sync=%d order_failed=%d consider=%u limit=%lu", __entry->nid, __print_symbolic(__entry->idx, ZONE_TYPE), __entry->order, + __entry->sync, __entry->order_failed, __entry->considered, 1UL << __entry->defer_shift) @@ -271,23 +274,23 @@ DECLARE_EVENT_CLASS(mm_compaction_defer_template, DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_deferred, - TP_PROTO(struct zone *zone, int order), + TP_PROTO(struct zone *zone, int order, bool sync), - TP_ARGS(zone, order) + TP_ARGS(zone, order, sync) ); DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_defer_compaction, - TP_PROTO(struct zone *zone, int order), + TP_PROTO(struct zone *zone, int order, bool sync), - TP_ARGS(zone, order) + TP_ARGS(zone, order, sync) ); DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_defer_reset, - TP_PROTO(struct zone *zone, int order), + TP_PROTO(struct zone *zone, int order, bool sync), - TP_ARGS(zone, order) + TP_ARGS(zone, order, sync) ); TRACE_EVENT(mm_compaction_kcompactd_sleep, diff --git a/mm/compaction.c b/mm/compaction.c index a049415512c67..7f8845d1990aa 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -124,35 +124,35 @@ static unsigned long release_free_list(struct list_head *freepages) * allocation success. 1 << compact_defer_shift, compactions are skipped up * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT */ -static void defer_compaction(struct zone *zone, int order) +static void defer_compaction(struct zone *zone, int order, bool sync) { - zone->compact_considered = 0; - zone->compact_defer_shift++; + zone->compact_considered[sync] = 0; + zone->compact_defer_shift[sync]++; - if (order < zone->compact_order_failed) - zone->compact_order_failed = order; + if (order < zone->compact_order_failed[sync]) + zone->compact_order_failed[sync] = order; - if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT) - zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; + if (zone->compact_defer_shift[sync] > COMPACT_MAX_DEFER_SHIFT) + zone->compact_defer_shift[sync] = COMPACT_MAX_DEFER_SHIFT; - trace_mm_compaction_defer_compaction(zone, order); + trace_mm_compaction_defer_compaction(zone, order, sync); } /* Returns true if compaction should be skipped this time */ -static bool compaction_deferred(struct zone *zone, int order) +static bool compaction_deferred(struct zone *zone, int order, bool sync) { - unsigned long defer_limit = 1UL << zone->compact_defer_shift; + unsigned long defer_limit = 1UL << zone->compact_defer_shift[sync]; - if (order < zone->compact_order_failed) + if (order < zone->compact_order_failed[sync]) return false; /* Avoid possible overflow */ - if (++zone->compact_considered >= defer_limit) { - zone->compact_considered = defer_limit; + if (++zone->compact_considered[sync] >= defer_limit) { + zone->compact_considered[sync] = defer_limit; return false; } - trace_mm_compaction_deferred(zone, order); + trace_mm_compaction_deferred(zone, order, sync); return true; } @@ -165,24 +165,28 @@ static bool compaction_deferred(struct zone *zone, int order) void compaction_defer_reset(struct zone *zone, int order, bool alloc_success) { - if (alloc_success) { - zone->compact_considered = 0; - zone->compact_defer_shift = 0; + int sync; + + for (sync = 0; sync < ASYNC_AND_SYNC; sync++) { + if (alloc_success) { + zone->compact_considered[sync] = 0; + zone->compact_defer_shift[sync] = 0; + } + if (order >= zone->compact_order_failed[sync]) + zone->compact_order_failed[sync] = order + 1; } - if (order >= zone->compact_order_failed) - zone->compact_order_failed = order + 1; - trace_mm_compaction_defer_reset(zone, order); + trace_mm_compaction_defer_reset(zone, order, true); } -/* Returns true if restarting compaction after many failures */ +/* Returns true if restarting sync compaction after many failures */ static bool compaction_restarting(struct zone *zone, int order) { - if (order < zone->compact_order_failed) + if (order < zone->compact_order_failed[true]) return false; - return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT && - zone->compact_considered >= 1UL << zone->compact_defer_shift; + return zone->compact_defer_shift[true] == COMPACT_MAX_DEFER_SHIFT && + zone->compact_considered[true] >= 1UL << zone->compact_defer_shift[true]; } /* Returns true if the pageblock should be scanned for pages to isolate. */ @@ -2855,7 +2859,7 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, continue; if (prio > MIN_COMPACT_PRIORITY - && compaction_deferred(zone, order)) { + && compaction_deferred(zone, order, true)) { rc = max_t(enum compact_result, COMPACT_DEFERRED, rc); continue; } @@ -2893,7 +2897,7 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, * so we defer compaction there. If it ends up * succeeding after all, it will be reset. */ - defer_compaction(zone, order); + defer_compaction(zone, order, true); /* * We might have stopped compacting due to need_resched() in @@ -3111,7 +3115,7 @@ static void kcompactd_do_work(pg_data_t *pgdat) if (!populated_zone(zone)) continue; - if (compaction_deferred(zone, cc.order)) + if (compaction_deferred(zone, cc.order, true)) continue; ret = compaction_suit_allocation_order(zone, @@ -3141,7 +3145,7 @@ static void kcompactd_do_work(pg_data_t *pgdat) * We use sync migration mode here, so we defer like * sync direct compaction does. */ - defer_compaction(zone, cc.order); + defer_compaction(zone, cc.order, true); } count_compact_events(KCOMPACTD_MIGRATE_SCANNED, -- 2.43.0
