Skip to content

K
Kernel

openeuler / Kernel
大约 2 年 前同步成功

通知 8
Star 0
Fork 0
K
Kernel
提交 d915169d 编写于 作者: M Mike Kravetz 提交者： Zheng Zengkai
浏览文件
操作

hugetlb: make free_huge_page irq safe 

mainline inclusion
from mainline-v5.13-rc1
commit db71ef79
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/I3ZCW9
CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=db71ef79b59bb2e78dc4df83d0e4bf6beaa5c82d

-------------------------------------------------

Commit c77c0a8a ("mm/hugetlb: defer freeing of huge pages if in
non-task context") was added to address the issue of free_huge_page being
called from irq context.  That commit hands off free_huge_page processing
to a workqueue if !in_task.  However, this doesn't cover all the cases as
pointed out by 0day bot lockdep report [1].

:  Possible interrupt unsafe locking scenario:
:
:        CPU0                    CPU1
:        ----                    ----
:   lock(hugetlb_lock);
:                                local_irq_disable();
:                                lock(slock-AF_INET);
:                                lock(hugetlb_lock);
:   <Interrupt>
:     lock(slock-AF_INET);

Shakeel has later explained that this is very likely TCP TX zerocopy from
hugetlb pages scenario when the networking code drops a last reference to
hugetlb page while having IRQ disabled.  Hugetlb freeing path doesn't
disable IRQ while holding hugetlb_lock so a lock dependency chain can lead
to a deadlock.

This commit addresses the issue by doing the following:
 - Make hugetlb_lock irq safe. This is mostly a simple process of
   changing spin_*lock calls to spin_*lock_irq* calls.
 - Make subpool lock irq safe in a similar manner.
 - Revert the !in_task check and workqueue handoff.

[1] https://lore.kernel.org/linux-mm/000000000000f1c03b05bc43aadc@google.com/

Link: https://lkml.kernel.org/r/20210409205254.242291-8-mike.kravetz@oracle.comSigned-off-by: NMike Kravetz <mike.kravetz@oracle.com>
Acked-by: NMichal Hocko <mhocko@suse.com>
Reviewed-by: NMuchun Song <songmuchun@bytedance.com>
Reviewed-by: NOscar Salvador <osalvador@suse.de>
Cc: "Aneesh Kumar K . V" <aneesh.kumar@linux.ibm.com>
Cc: Barry Song <song.bao.hua@hisilicon.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hillf Danton <hdanton@sina.com>
Cc: HORIGUCHI NAOYA <naoya.horiguchi@nec.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Mina Almasry <almasrymina@google.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Roman Gushchin <guro@fb.com>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Waiman Long <longman@redhat.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: NAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: NLinus Torvalds <torvalds@linux-foundation.org>

Conflicts:
	mm/hugetlb_cgroup.c
	mm/hugetlb.c
Signed-off-by: NChen Huang <chenhuang5@huawei.com>
Signed-off-by: NNanyong Sun <sunnanyong@huawei.com>
Reviewed-by: NTong Tiangen <tongtiangen@huawei.com>
Signed-off-by: NZheng Zengkai <zhengzengkai@huawei.com>
上级 843f2eb4
隐藏空白更改
内联 并排
Showing with 67 addition and 110 deletion
mm/hugetlb.c
浏览文件 @ d915169d
...@@ -82,11 +82,12 @@ struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; ...@@ -82,11 +82,12 @@ struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
/* Forward declaration */ /* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta); static int hugetlb_acct_memory(struct hstate *h, long delta);
static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
unsigned long irq_flags)
{ {
bool free = (spool->count == 0) && (spool->used_hpages == 0); bool free = (spool->count == 0) && (spool->used_hpages == 0);
spin_unlock(&spool->lock); spin_unlock_irqrestore(&spool->lock, irq_flags);
/* If no pages are used, and no other handles to the subpool /* If no pages are used, and no other handles to the subpool
* remain, give up any reservations based on minimum size and * remain, give up any reservations based on minimum size and
...@@ -125,10 +126,12 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, ...@@ -125,10 +126,12 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
void hugepage_put_subpool(struct hugepage_subpool *spool) void hugepage_put_subpool(struct hugepage_subpool *spool)
{ {
spin_lock(&spool->lock); unsigned long flags;
spin_lock_irqsave(&spool->lock, flags);
BUG_ON(!spool->count); BUG_ON(!spool->count);
spool->count--; spool->count--;
unlock_or_release_subpool(spool); unlock_or_release_subpool(spool, flags);
} }
/* /*
...@@ -147,7 +150,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, ...@@ -147,7 +150,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
if (!spool) if (!spool)
return ret; return ret;
spin_lock(&spool->lock); spin_lock_irq(&spool->lock);
if (spool->max_hpages != -1) { /* maximum size accounting */ if (spool->max_hpages != -1) { /* maximum size accounting */
if ((spool->used_hpages + delta) <= spool->max_hpages) if ((spool->used_hpages + delta) <= spool->max_hpages)
...@@ -174,7 +177,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, ...@@ -174,7 +177,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
} }
unlock_ret: unlock_ret:
spin_unlock(&spool->lock); spin_unlock_irq(&spool->lock);
return ret; return ret;
} }
...@@ -188,11 +191,12 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, ...@@ -188,11 +191,12 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
long delta) long delta)
{ {
long ret = delta; long ret = delta;
unsigned long flags;
if (!spool) if (!spool)
return delta; return delta;
spin_lock(&spool->lock); spin_lock_irqsave(&spool->lock, flags);
if (spool->max_hpages != -1) /* maximum size accounting */ if (spool->max_hpages != -1) /* maximum size accounting */
spool->used_hpages -= delta; spool->used_hpages -= delta;
...@@ -213,7 +217,7 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, ...@@ -213,7 +217,7 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
* If hugetlbfs_put_super couldn't free spool due to an outstanding * If hugetlbfs_put_super couldn't free spool due to an outstanding
* quota reference, free it now. * quota reference, free it now.
*/ */
unlock_or_release_subpool(spool); unlock_or_release_subpool(spool, flags);
return ret; return ret;
} }
...@@ -1401,7 +1405,7 @@ struct hstate *size_to_hstate(unsigned long size) ...@@ -1401,7 +1405,7 @@ struct hstate *size_to_hstate(unsigned long size)
return NULL; return NULL;
} }
static void __free_huge_page(struct page *page) void free_huge_page(struct page *page)
{ {
/* /*
* Can't pass hstate in here because it is called from the * Can't pass hstate in here because it is called from the
...@@ -1411,6 +1415,7 @@ static void __free_huge_page(struct page *page) ...@@ -1411,6 +1415,7 @@ static void __free_huge_page(struct page *page)
int nid = page_to_nid(page); int nid = page_to_nid(page);
struct hugepage_subpool *spool = hugetlb_page_subpool(page); struct hugepage_subpool *spool = hugetlb_page_subpool(page);
bool restore_reserve; bool restore_reserve;
unsigned long flags;
VM_BUG_ON_PAGE(page_count(page), page); VM_BUG_ON_PAGE(page_count(page), page);
VM_BUG_ON_PAGE(page_mapcount(page), page); VM_BUG_ON_PAGE(page_mapcount(page), page);
...@@ -1439,7 +1444,7 @@ static void __free_huge_page(struct page *page) ...@@ -1439,7 +1444,7 @@ static void __free_huge_page(struct page *page)
restore_reserve = true; restore_reserve = true;
} }
spin_lock(&hugetlb_lock); spin_lock_irqsave(&hugetlb_lock, flags);
ClearHPageMigratable(page); ClearHPageMigratable(page);
hugetlb_cgroup_uncharge_page(hstate_index(h), hugetlb_cgroup_uncharge_page(hstate_index(h),
pages_per_huge_page(h), page); pages_per_huge_page(h), page);
...@@ -1450,66 +1455,18 @@ static void __free_huge_page(struct page *page) ...@@ -1450,66 +1455,18 @@ static void __free_huge_page(struct page *page)
if (HPageTemporary(page)) { if (HPageTemporary(page)) {
remove_hugetlb_page(h, page, false); remove_hugetlb_page(h, page, false);
spin_unlock(&hugetlb_lock); spin_unlock_irqrestore(&hugetlb_lock, flags);
update_and_free_page(h, page); update_and_free_page(h, page);
} else if (h->surplus_huge_pages_node[nid]) { } else if (h->surplus_huge_pages_node[nid]) {
/* remove the page from active list */ /* remove the page from active list */
remove_hugetlb_page(h, page, true); remove_hugetlb_page(h, page, true);
spin_unlock(&hugetlb_lock); spin_unlock_irqrestore(&hugetlb_lock, flags);
update_and_free_page(h, page); update_and_free_page(h, page);
} else { } else {
arch_clear_hugepage_flags(page); arch_clear_hugepage_flags(page);
enqueue_huge_page(h, page); enqueue_huge_page(h, page);
spin_unlock(&hugetlb_lock); spin_unlock_irqrestore(&hugetlb_lock, flags);
}
}
/*
* As free_huge_page() can be called from a non-task context, we have
* to defer the actual freeing in a workqueue to prevent potential
* hugetlb_lock deadlock.
*
* free_hpage_workfn() locklessly retrieves the linked list of pages to
* be freed and frees them one-by-one. As the page->mapping pointer is
* going to be cleared in __free_huge_page() anyway, it is reused as the
* llist_node structure of a lockless linked list of huge pages to be freed.
*/
static LLIST_HEAD(hpage_freelist);
static void free_hpage_workfn(struct work_struct *work)
{
struct llist_node *node;
struct page *page;
node = llist_del_all(&hpage_freelist);
while (node) {
page = container_of((struct address_space **)node,
struct page, mapping);
node = node->next;
__free_huge_page(page);
}
}
static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
void free_huge_page(struct page *page)
{
/*
* Defer freeing if in non-task context to avoid hugetlb_lock deadlock.
*/
if (!in_task()) {
/*
* Only call schedule_work() if hpage_freelist is previously
* empty. Otherwise, schedule_work() had been called but the
* workfn hasn't retrieved the list yet.
*/
if (llist_add((struct llist_node *)&page->mapping,
&hpage_freelist))
schedule_work(&free_hpage_work);
return;
} }
__free_huge_page(page);
} }
static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
...@@ -1518,11 +1475,11 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) ...@@ -1518,11 +1475,11 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
set_hugetlb_cgroup(page, NULL); set_hugetlb_cgroup(page, NULL);
set_hugetlb_cgroup_rsvd(page, NULL); set_hugetlb_cgroup_rsvd(page, NULL);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
h->nr_huge_pages++; h->nr_huge_pages++;
h->nr_huge_pages_node[nid]++; h->nr_huge_pages_node[nid]++;
ClearHPageFreed(page); ClearHPageFreed(page);
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
} }
static void prep_compound_gigantic_page(struct page *page, unsigned int order) static void prep_compound_gigantic_page(struct page *page, unsigned int order)
...@@ -1770,7 +1727,7 @@ int dissolve_free_huge_page(struct page *page) ...@@ -1770,7 +1727,7 @@ int dissolve_free_huge_page(struct page *page)
if (!PageHuge(page)) if (!PageHuge(page))
return 0; return 0;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
if (!PageHuge(page)) { if (!PageHuge(page)) {
rc = 0; rc = 0;
goto out; goto out;
...@@ -1787,7 +1744,7 @@ int dissolve_free_huge_page(struct page *page) ...@@ -1787,7 +1744,7 @@ int dissolve_free_huge_page(struct page *page)
* when it is dissolved. * when it is dissolved.
*/ */
if (unlikely(!HPageFreed(head))) { if (unlikely(!HPageFreed(head))) {
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
cond_resched(); cond_resched();
/* /*
...@@ -1811,12 +1768,12 @@ int dissolve_free_huge_page(struct page *page) ...@@ -1811,12 +1768,12 @@ int dissolve_free_huge_page(struct page *page)
} }
remove_hugetlb_page(h, page, false); remove_hugetlb_page(h, page, false);
h->max_huge_pages--; h->max_huge_pages--;
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
update_and_free_page(h, head); update_and_free_page(h, head);
return 0; return 0;
} }
out: out:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return rc; return rc;
} }
...@@ -1858,16 +1815,16 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, ...@@ -1858,16 +1815,16 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
if (hstate_is_gigantic(h)) if (hstate_is_gigantic(h))
return NULL; return NULL;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
goto out_unlock; goto out_unlock;
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
if (!page) if (!page)
return NULL; return NULL;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
/* /*
* We could have raced with the pool size change. * We could have raced with the pool size change.
* Double check that and simply deallocate the new page * Double check that and simply deallocate the new page
...@@ -1877,7 +1834,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, ...@@ -1877,7 +1834,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
*/ */
if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
SetHPageTemporary(page); SetHPageTemporary(page);
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
put_page(page); put_page(page);
return NULL; return NULL;
} else { } else {
...@@ -1886,7 +1843,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, ...@@ -1886,7 +1843,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
} }
out_unlock: out_unlock:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return page; return page;
} }
...@@ -1936,17 +1893,17 @@ struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h, ...@@ -1936,17 +1893,17 @@ struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
nodemask_t *nmask, gfp_t gfp_mask) nodemask_t *nmask, gfp_t gfp_mask)
{ {
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
if (h->free_huge_pages - h->resv_huge_pages > 0) { if (h->free_huge_pages - h->resv_huge_pages > 0) {
struct page *page; struct page *page;
page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask); page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
if (page) { if (page) {
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return page; return page;
} }
} }
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask); return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
} }
...@@ -1993,7 +1950,7 @@ static int gather_surplus_pages(struct hstate *h, int delta) ...@@ -1993,7 +1950,7 @@ static int gather_surplus_pages(struct hstate *h, int delta)
ret = -ENOMEM; ret = -ENOMEM;
retry: retry:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
for (i = 0; i < needed; i++) { for (i = 0; i < needed; i++) {
page = alloc_surplus_huge_page(h, htlb_alloc_mask(h), page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
NUMA_NO_NODE, NULL); NUMA_NO_NODE, NULL);
...@@ -2010,7 +1967,7 @@ static int gather_surplus_pages(struct hstate *h, int delta) ...@@ -2010,7 +1967,7 @@ static int gather_surplus_pages(struct hstate *h, int delta)
* After retaking hugetlb_lock, we need to recalculate 'needed' * After retaking hugetlb_lock, we need to recalculate 'needed'
* because either resv_huge_pages or free_huge_pages may have changed. * because either resv_huge_pages or free_huge_pages may have changed.
*/ */
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
needed = (h->resv_huge_pages + delta) - needed = (h->resv_huge_pages + delta) -
(h->free_huge_pages + allocated); (h->free_huge_pages + allocated);
if (needed > 0) { if (needed > 0) {
...@@ -2048,12 +2005,12 @@ static int gather_surplus_pages(struct hstate *h, int delta) ...@@ -2048,12 +2005,12 @@ static int gather_surplus_pages(struct hstate *h, int delta)
enqueue_huge_page(h, page); enqueue_huge_page(h, page);
} }
free: free:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
/* Free unnecessary surplus pages to the buddy allocator */ /* Free unnecessary surplus pages to the buddy allocator */
list_for_each_entry_safe(page, tmp, &surplus_list, lru) list_for_each_entry_safe(page, tmp, &surplus_list, lru)
put_page(page); put_page(page);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
return ret; return ret;
} }
...@@ -2103,9 +2060,9 @@ static void return_unused_surplus_pages(struct hstate *h, ...@@ -2103,9 +2060,9 @@ static void return_unused_surplus_pages(struct hstate *h,
} }
out: out:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
update_and_free_pages_bulk(h, &page_list); update_and_free_pages_bulk(h, &page_list);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
} }
...@@ -2340,7 +2297,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, ...@@ -2340,7 +2297,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,
if (ret) if (ret)
goto out_uncharge_cgroup_reservation; goto out_uncharge_cgroup_reservation;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
/* /*
* glb_chg is passed to indicate whether or not a page must be taken * glb_chg is passed to indicate whether or not a page must be taken
* from the global free pool (global change). gbl_chg == 0 indicates * from the global free pool (global change). gbl_chg == 0 indicates
...@@ -2348,7 +2305,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, ...@@ -2348,7 +2305,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,
*/ */
page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
if (!page) { if (!page) {
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
page = alloc_buddy_huge_page_with_mpol(h, vma, addr); page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
if (!page) if (!page)
goto out_uncharge_cgroup; goto out_uncharge_cgroup;
...@@ -2356,7 +2313,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, ...@@ -2356,7 +2313,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,
SetHPageRestoreReserve(page); SetHPageRestoreReserve(page);
h->resv_huge_pages--; h->resv_huge_pages--;
} }
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
list_add(&page->lru, &h->hugepage_activelist); list_add(&page->lru, &h->hugepage_activelist);
/* Fall through */ /* Fall through */
} }
...@@ -2369,7 +2326,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, ...@@ -2369,7 +2326,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,
h_cg, page); h_cg, page);
} }
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
hugetlb_set_page_subpool(page, spool); hugetlb_set_page_subpool(page, spool);
...@@ -2581,9 +2538,9 @@ static void try_to_free_low(struct hstate *h, unsigned long count, ...@@ -2581,9 +2538,9 @@ static void try_to_free_low(struct hstate *h, unsigned long count,
} }
out: out:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
update_and_free_pages_bulk(h, &page_list); update_and_free_pages_bulk(h, &page_list);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
} }
#else #else
static inline void try_to_free_low(struct hstate *h, unsigned long count, static inline void try_to_free_low(struct hstate *h, unsigned long count,
...@@ -2648,7 +2605,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, ...@@ -2648,7 +2605,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
* pages in hstate via the proc/sysfs interfaces. * pages in hstate via the proc/sysfs interfaces.
*/ */
mutex_lock(&h->resize_lock); mutex_lock(&h->resize_lock);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
/* /*
* Check for a node specific request. * Check for a node specific request.
...@@ -2679,7 +2636,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, ...@@ -2679,7 +2636,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
*/ */
if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) { if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
if (count > persistent_huge_pages(h)) { if (count > persistent_huge_pages(h)) {
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
mutex_unlock(&h->resize_lock); mutex_unlock(&h->resize_lock);
NODEMASK_FREE(node_alloc_noretry); NODEMASK_FREE(node_alloc_noretry);
return -EINVAL; return -EINVAL;
...@@ -2709,14 +2666,14 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, ...@@ -2709,14 +2666,14 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
* page, free_huge_page will handle it by freeing the page * page, free_huge_page will handle it by freeing the page
* and reducing the surplus. * and reducing the surplus.
*/ */
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
/* yield cpu to avoid soft lockup */ /* yield cpu to avoid soft lockup */
cond_resched(); cond_resched();
ret = alloc_pool_huge_page(h, nodes_allowed, ret = alloc_pool_huge_page(h, nodes_allowed,
node_alloc_noretry); node_alloc_noretry);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
if (!ret) if (!ret)
goto out; goto out;
...@@ -2755,9 +2712,9 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, ...@@ -2755,9 +2712,9 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
list_add(&page->lru, &page_list); list_add(&page->lru, &page_list);
} }
/* free the pages after dropping lock */ /* free the pages after dropping lock */
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
update_and_free_pages_bulk(h, &page_list); update_and_free_pages_bulk(h, &page_list);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
while (count < persistent_huge_pages(h)) { while (count < persistent_huge_pages(h)) {
if (!adjust_pool_surplus(h, nodes_allowed, 1)) if (!adjust_pool_surplus(h, nodes_allowed, 1))
...@@ -2765,7 +2722,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, ...@@ -2765,7 +2722,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
} }
out: out:
h->max_huge_pages = persistent_huge_pages(h); h->max_huge_pages = persistent_huge_pages(h);
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
mutex_unlock(&h->resize_lock); mutex_unlock(&h->resize_lock);
NODEMASK_FREE(node_alloc_noretry); NODEMASK_FREE(node_alloc_noretry);
...@@ -2920,9 +2877,9 @@ static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, ...@@ -2920,9 +2877,9 @@ static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
if (err) if (err)
return err; return err;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
h->nr_overcommit_huge_pages = input; h->nr_overcommit_huge_pages = input;
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return count; return count;
} }
...@@ -3512,9 +3469,9 @@ int hugetlb_overcommit_handler(struct ctl_table *table, int write, ...@@ -3512,9 +3469,9 @@ int hugetlb_overcommit_handler(struct ctl_table *table, int write,
goto out; goto out;
if (write) { if (write) {
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
h->nr_overcommit_huge_pages = tmp; h->nr_overcommit_huge_pages = tmp;
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
} }
out: out:
return ret; return ret;
...@@ -3607,7 +3564,7 @@ static int hugetlb_acct_memory(struct hstate *h, long delta) ...@@ -3607,7 +3564,7 @@ static int hugetlb_acct_memory(struct hstate *h, long delta)
{ {
int ret = -ENOMEM; int ret = -ENOMEM;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
/* /*
* When cpuset is configured, it breaks the strict hugetlb page * When cpuset is configured, it breaks the strict hugetlb page
* reservation as the accounting is done on a global variable. Such * reservation as the accounting is done on a global variable. Such
...@@ -3646,7 +3603,7 @@ static int hugetlb_acct_memory(struct hstate *h, long delta) ...@@ -3646,7 +3603,7 @@ static int hugetlb_acct_memory(struct hstate *h, long delta)
return_unused_surplus_pages(h, (unsigned long) -delta); return_unused_surplus_pages(h, (unsigned long) -delta);
out: out:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return ret; return ret;
} }
...@@ -5638,7 +5595,7 @@ bool isolate_huge_page(struct page *page, struct list_head *list) ...@@ -5638,7 +5595,7 @@ bool isolate_huge_page(struct page *page, struct list_head *list)
{ {
bool ret = true; bool ret = true;
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
if (!PageHeadHuge(page) || if (!PageHeadHuge(page) ||
!HPageMigratable(page) || !HPageMigratable(page) ||
!get_page_unless_zero(page)) { !get_page_unless_zero(page)) {
...@@ -5648,17 +5605,17 @@ bool isolate_huge_page(struct page *page, struct list_head *list) ...@@ -5648,17 +5605,17 @@ bool isolate_huge_page(struct page *page, struct list_head *list)
ClearHPageMigratable(page); ClearHPageMigratable(page);
list_move_tail(&page->lru, list); list_move_tail(&page->lru, list);
unlock: unlock:
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return ret; return ret;
} }
void putback_active_hugepage(struct page *page) void putback_active_hugepage(struct page *page)
{ {
VM_BUG_ON_PAGE(!PageHead(page), page); VM_BUG_ON_PAGE(!PageHead(page), page);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
SetHPageMigratable(page); SetHPageMigratable(page);
list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
put_page(page); put_page(page);
} }
...@@ -5686,12 +5643,12 @@ void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) ...@@ -5686,12 +5643,12 @@ void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
SetHPageTemporary(oldpage); SetHPageTemporary(oldpage);
ClearHPageTemporary(newpage); ClearHPageTemporary(newpage);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
if (h->surplus_huge_pages_node[old_nid]) { if (h->surplus_huge_pages_node[old_nid]) {
h->surplus_huge_pages_node[old_nid]--; h->surplus_huge_pages_node[old_nid]--;
h->surplus_huge_pages_node[new_nid]++; h->surplus_huge_pages_node[new_nid]++;
} }
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
} }
} }
......
mm/hugetlb_cgroup.c
浏览文件 @ d915169d
...@@ -204,11 +204,11 @@ static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css) ...@@ -204,11 +204,11 @@ static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css)
do { do {
idx = 0; idx = 0;
for_each_hstate(h) { for_each_hstate(h) {
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
list_for_each_entry(page, &h->hugepage_activelist, lru) list_for_each_entry(page, &h->hugepage_activelist, lru)
hugetlb_cgroup_move_parent(idx, h_cg, page); hugetlb_cgroup_move_parent(idx, h_cg, page);
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
idx++; idx++;
} }
cond_resched(); cond_resched();
...@@ -785,7 +785,7 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) ...@@ -785,7 +785,7 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage)
return; return;
VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage); VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage);
spin_lock(&hugetlb_lock); spin_lock_irq(&hugetlb_lock);
h_cg = hugetlb_cgroup_from_page(oldhpage); h_cg = hugetlb_cgroup_from_page(oldhpage);
h_cg_rsvd = hugetlb_cgroup_from_page_rsvd(oldhpage); h_cg_rsvd = hugetlb_cgroup_from_page_rsvd(oldhpage);
set_hugetlb_cgroup(oldhpage, NULL); set_hugetlb_cgroup(oldhpage, NULL);
...@@ -795,7 +795,7 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) ...@@ -795,7 +795,7 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage)
set_hugetlb_cgroup(newhpage, h_cg); set_hugetlb_cgroup(newhpage, h_cg);
set_hugetlb_cgroup_rsvd(newhpage, h_cg_rsvd); set_hugetlb_cgroup_rsvd(newhpage, h_cg_rsvd);
list_move(&newhpage->lru, &h->hugepage_activelist); list_move(&newhpage->lru, &h->hugepage_activelist);
spin_unlock(&hugetlb_lock); spin_unlock_irq(&hugetlb_lock);
return; return;
} }
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或