diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h index 65d4b5dbf3f6b7e765c06225f5673d7914a63b20..05bc55ef831b088e32e8cb12d9592441136dd96b 100644 --- a/include/linux/dynamic_hugetlb.h +++ b/include/linux/dynamic_hugetlb.h @@ -102,6 +102,7 @@ bool file_has_mem_in_hpool(struct hugetlbfs_inode_info *p); int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_info *p); struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetlb_pool *hpool, bool need_unreserved); +void free_huge_page_to_dhugetlb_pool(struct page *page, bool restore_reserve); #else @@ -162,6 +163,10 @@ struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetl { return NULL; } +static inline +void free_huge_page_to_dhugetlb_pool(struct page *page, bool restore_reserve) +{ +} #endif #endif /* CONFIG_DYNAMIC_HUGETLB */ diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 126b3d9d375473547d40114be973583cc7b0b30d..73795d533f7e24548cc56d410efc528a45c154d1 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -649,6 +649,36 @@ struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetl return page; } +void free_huge_page_to_dhugetlb_pool(struct page *page, bool restore_reserve) +{ + struct hstate *h = page_hstate(page); + struct huge_pages_pool *hpages_pool; + struct dhugetlb_pool *hpool; + + hpool = find_hpool_by_dhugetlb_pagelist(page); + + if (!get_hpool_unless_zero(hpool)) { + pr_err("dhugetlb: free error: get hpool failed\n"); + return; + } + + spin_lock(&hpool->lock); + ClearPagePool(page); + set_compound_page_dtor(page, NULL_COMPOUND_DTOR); + if (hstate_is_gigantic(h)) + hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_1G]; + else + hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_2M]; + + list_add(&page->lru, &hpages_pool->hugepage_freelists); + hpages_pool->free_huge_pages++; + hpages_pool->used_huge_pages--; + if (restore_reserve) + hpages_pool->resv_huge_pages++; + spin_unlock(&hpool->lock); + put_hpool(hpool); +} + static int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, unsigned long nid, unsigned long nr_pages) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 031ad320f10ce8bb1079ee64ab7b5cb8eb7015ac..3b787cb5669907c1035332f00d70826f15c59d85 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1583,6 +1583,19 @@ void free_huge_page(struct page *page) restore_reserve = HPageRestoreReserve(page); ClearHPageRestoreReserve(page); + if (dhugetlb_enabled && PagePool(page)) { + spin_lock(&hugetlb_lock); + ClearHPageMigratable(page); + list_del(&page->lru); + hugetlb_cgroup_uncharge_page(hstate_index(h), + pages_per_huge_page(h), page); + hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), + pages_per_huge_page(h), page); + spin_unlock(&hugetlb_lock); + free_huge_page_to_dhugetlb_pool(page, restore_reserve); + return; + } + /* * If HPageRestoreReserve was set on page, page allocation consumed a * reservation. If the page was associated with a subpool, there