anierin/msm-5.15
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BACKPORT: mm/page_alloc: protect PCP lists with a spinlock

Browse Source 
Currently the PCP lists are protected by using local_lock_irqsave to
prevent migration and IRQ reentrancy but this is inconvenient.  Remote
draining of the lists is impossible and a workqueue is required and every
task allocation/free must disable then enable interrupts which is
expensive.

As preparation for dealing with both of those problems, protect the
lists with a spinlock.  The IRQ-unsafe version of the lock is used
because IRQs are already disabled by local_lock_irqsave.  spin_trylock
is used in combination with local_lock_irqsave() but later will be
replaced with a spin_trylock_irqsave when the local_lock is removed.

The per_cpu_pages still fits within the same number of cache lines after
this patch relative to before the series.

struct per_cpu_pages {
        spinlock_t                 lock;                 /*     0     4 */
        int                        count;                /*     4     4 */
        int                        high;                 /*     8     4 */
        int                        batch;                /*    12     4 */
        short int                  free_factor;          /*    16     2 */
        short int                  expire;               /*    18     2 */

        /* XXX 4 bytes hole, try to pack */

        struct list_head           lists[13];            /*    24   208 */

        /* size: 256, cachelines: 4, members: 7 */
        /* sum members: 228, holes: 1, sum holes: 4 */
        /* padding: 24 */
} __attribute__((__aligned__(64)));

There is overhead in the fast path due to acquiring the spinlock even
though the spinlock is per-cpu and uncontended in the common case.  Page
Fault Test (PFT) running on a 1-socket reported the following results on a
1 socket machine.

                                     5.19.0-rc3               5.19.0-rc3
                                        vanilla      mm-pcpspinirq-v5r16
Hmean     faults/sec-1   869275.7381 (   0.00%)   874597.5167 *   0.61%*
Hmean     faults/sec-3  2370266.6681 (   0.00%)  2379802.0362 *   0.40%*
Hmean     faults/sec-5  2701099.7019 (   0.00%)  2664889.7003 *  -1.34%*
Hmean     faults/sec-7  3517170.9157 (   0.00%)  3491122.8242 *  -0.74%*
Hmean     faults/sec-8  3965729.6187 (   0.00%)  3939727.0243 *  -0.66%*

There is a small hit in the number of faults per second but given that the
results are more stable, it's borderline noise.

[akpm@linux-foundation.org: add missing local_unlock_irqrestore() on contention path]
Link: https://lkml.kernel.org/r/20220624125423.6126-6-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Tested-by: Yu Zhao <yuzhao@google.com>
Reviewed-by: Nicolas Saenz Julienne <nsaenzju@redhat.com>
Tested-by: Nicolas Saenz Julienne <nsaenzju@redhat.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Bug: 230899966
(cherry picked from commit 4b23a68f953628eb4e4b7fe1294ebf93d4b8ceee)
[surenb: resolved merge conflicts due to the differences with 6.0]
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Change-Id: I03ff1c22301e7f8735947e71413376ea143e855a
This commit is contained in:
Mel Gorman
2022-06-24 13:54:21 +01:00
committed by Suren Baghdasaryan
parent bd093f8791
commit cf666fb569
2 changed files with 98 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
include/linux/mmzone.h
View File

@@ -571,6 +571,7 @@ enum zone_watermarks {
/* Fields and list protected by pagesets local_lock in page_alloc.c */
struct per_cpu_pages {
spinlock_t lock; /* Protects lists field */
int count; /* number of pages in the list */
int high; /* high watermark, emptying needed */
int batch; /* chunk size for buddy add/remove */

117
mm/page_alloc.c
View File

@@ -131,6 +131,20 @@ static DEFINE_PER_CPU(struct pagesets, pagesets) = {
.lock = INIT_LOCAL_LOCK(lock),
};
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
/*
* On SMP, spin_trylock is sufficient protection.
* On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP.
*/
#define pcp_trylock_prepare(flags) do { } while (0)
#define pcp_trylock_finish(flag) do { } while (0)
#else
/* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */
#define pcp_trylock_prepare(flags) local_irq_save(flags)
#define pcp_trylock_finish(flags) local_irq_restore(flags)
#endif
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);
EXPORT_PER_CPU_SYMBOL(numa_node);
@@ -3195,15 +3209,22 @@ static struct list_head *get_populated_pcp_list(struct zone *zone,
*/
void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
{
unsigned long flags;
int to_drain, batch;
local_lock_irqsave(&pagesets.lock, flags);
batch = READ_ONCE(pcp->batch);
to_drain = min(pcp->count, batch);
if (to_drain > 0)
if (to_drain > 0) {
unsigned long flags;
/*
* free_pcppages_bulk expects IRQs disabled for zone->lock
* so even though pcp->lock is not intended to be IRQ-safe,
* it's needed in this context.
*/
spin_lock_irqsave(&pcp->lock, flags);
free_pcppages_bulk(zone, to_drain, pcp);
local_unlock_irqrestore(&pagesets.lock, flags);
spin_unlock_irqrestore(&pcp->lock, flags);
}
}
#endif
@@ -3216,16 +3237,17 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
*/
static void drain_pages_zone(unsigned int cpu, struct zone *zone)
{
unsigned long flags;
struct per_cpu_pages *pcp;
local_lock_irqsave(&pagesets.lock, flags);
pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
if (pcp->count)
free_pcppages_bulk(zone, pcp->count, pcp);
if (pcp->count) {
unsigned long flags;
local_unlock_irqrestore(&pagesets.lock, flags);
/* See drain_zone_pages on why this is disabling IRQs */
spin_lock_irqsave(&pcp->lock, flags);
free_pcppages_bulk(zone, pcp->count, pcp);
spin_unlock_irqrestore(&pcp->lock, flags);
}
}
/*
@@ -3487,16 +3509,14 @@ static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone)
return min(READ_ONCE(pcp->batch) << 2, high);
}
static void free_unref_page_commit(struct page *page, unsigned long pfn,
static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp,
struct page *page, unsigned long pfn,
int migratetype, unsigned int order)
{
struct zone *zone = page_zone(page);
struct per_cpu_pages *pcp;
int high;
int pindex;
__count_vm_event(PGFREE);
pcp = this_cpu_ptr(zone->per_cpu_pageset);
pindex = order_to_pindex(migratetype, order);
list_add(&page->pcp_list, &pcp->lists[pindex]);
pcp->count += 1 << order;
@@ -3514,6 +3534,9 @@ static void free_unref_page_commit(struct page *page, unsigned long pfn,
void free_unref_page(struct page *page, unsigned int order)
{
unsigned long flags;
unsigned long __maybe_unused UP_flags;
struct per_cpu_pages *pcp;
struct zone *zone;
unsigned long pfn = page_to_pfn(page);
int migratetype;
bool pcp_skip_cma_pages = false;
@@ -3541,7 +3564,16 @@ void free_unref_page(struct page *page, unsigned int order)
}
local_lock_irqsave(&pagesets.lock, flags);
free_unref_page_commit(page, pfn, migratetype, order);
zone = page_zone(page);
pcp_trylock_prepare(UP_flags);
pcp = this_cpu_ptr(zone->per_cpu_pageset);
if (spin_trylock(&pcp->lock)) {
free_unref_page_commit(zone, pcp, page, pfn, migratetype, order);
spin_unlock(&pcp->lock);
} else {
free_one_page(zone, page, pfn, order, migratetype, FPI_NONE);
}
pcp_trylock_finish(UP_flags);
local_unlock_irqrestore(&pagesets.lock, flags);
}
@@ -3551,6 +3583,8 @@ void free_unref_page(struct page *page, unsigned int order)
void free_unref_page_list(struct list_head *list)
{
struct page *page, *next;
struct per_cpu_pages *pcp = NULL;
struct zone *locked_zone = NULL;
unsigned long flags, pfn;
int batch_count = 0;
int migratetype;
@@ -3583,6 +3617,17 @@ void free_unref_page_list(struct list_head *list)
local_lock_irqsave(&pagesets.lock, flags);
list_for_each_entry_safe(page, next, list, lru) {
struct zone *zone = page_zone(page);
/* Different zone, different pcp lock. */
if (zone != locked_zone) {
if (pcp)
spin_unlock(&pcp->lock);
locked_zone = zone;
pcp = this_cpu_ptr(zone->per_cpu_pageset);
spin_lock(&pcp->lock);
}
pfn = page_private(page);
set_page_private(page, 0);
@@ -3595,18 +3640,24 @@ void free_unref_page_list(struct list_head *list)
migratetype = MIGRATE_MOVABLE;
trace_mm_page_free_batched(page);
free_unref_page_commit(page, pfn, migratetype, 0);
free_unref_page_commit(zone, pcp, page, pfn, migratetype, 0);
/*
* Guard against excessive IRQ disabled times when we get
* a large list of pages to free.
*/
if (++batch_count == SWAP_CLUSTER_MAX) {
spin_unlock(&pcp->lock);
local_unlock_irqrestore(&pagesets.lock, flags);
batch_count = 0;
local_lock_irqsave(&pagesets.lock, flags);
pcp = this_cpu_ptr(locked_zone->per_cpu_pageset);
spin_lock(&pcp->lock);
}
}
if (pcp)
spin_unlock(&pcp->lock);
local_unlock_irqrestore(&pagesets.lock, flags);
}
@@ -3817,17 +3868,31 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
struct per_cpu_pages *pcp;
struct page *page;
unsigned long flags;
unsigned long __maybe_unused UP_flags;
local_lock_irqsave(&pagesets.lock, flags);
/*
* spin_trylock may fail due to a parallel drain. In the future, the
* trylock will also protect against IRQ reentrancy.
*/
pcp = this_cpu_ptr(zone->per_cpu_pageset);
pcp_trylock_prepare(UP_flags);
if (!spin_trylock(&pcp->lock)) {
pcp_trylock_finish(UP_flags);
local_unlock_irqrestore(&pagesets.lock, flags);
return NULL;
}
/*
* On allocation, reduce the number of pages that are batch freed.
* See nr_pcp_free() where free_factor is increased for subsequent
* frees.
*/
pcp = this_cpu_ptr(zone->per_cpu_pageset);
pcp->free_factor >>= 1;
page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, gfp_flags);
spin_unlock(&pcp->lock);
pcp_trylock_finish(UP_flags);
local_unlock_irqrestore(&pagesets.lock, flags);
if (page) {
__count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
@@ -3850,7 +3915,8 @@ struct page *rmqueue(struct zone *preferred_zone,
if (likely(pcp_allowed_order(order))) {
page = rmqueue_pcplist(preferred_zone, zone, order,
gfp_flags, migratetype, alloc_flags);
goto out;
if (likely(page))
goto out;
}
/*
@@ -5400,6 +5466,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
{
struct page *page;
unsigned long flags;
unsigned long __maybe_unused UP_flags;
struct zone *zone;
struct zoneref *z;
struct per_cpu_pages *pcp;
@@ -5479,10 +5546,14 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
if (unlikely(!zone))
goto failed;
/* Attempt the batch allocation */
/* Is a parallel drain in progress? */
local_lock_irqsave(&pagesets.lock, flags);
pcp_trylock_prepare(UP_flags);
pcp = this_cpu_ptr(zone->per_cpu_pageset);
if (!spin_trylock(&pcp->lock))
goto failed_irq;
/* Attempt the batch allocation */
while (nr_populated < nr_pages) {
/* Skip existing pages */
@@ -5495,8 +5566,10 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
pcp, alloc_gfp);
if (unlikely(!page)) {
/* Try and allocate at least one page */
if (!nr_account)
if (!nr_account) {
spin_unlock(&pcp->lock);
goto failed_irq;
}
break;
}
nr_account++;
@@ -5509,6 +5582,8 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
nr_populated++;
}
spin_unlock(&pcp->lock);
pcp_trylock_finish(UP_flags);
local_unlock_irqrestore(&pagesets.lock, flags);
__count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);
@@ -5518,6 +5593,7 @@ out:
return nr_populated;
failed_irq:
pcp_trylock_finish(UP_flags);
local_unlock_irqrestore(&pagesets.lock, flags);
failed:
@@ -7097,6 +7173,7 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta
memset(pcp, 0, sizeof(*pcp));
memset(pzstats, 0, sizeof(*pzstats));
spin_lock_init(&pcp->lock);
for (pindex = 0; pindex < NR_PCP_LISTS; pindex++)
INIT_LIST_HEAD(&pcp->lists[pindex]);