diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 246858ea0a525054da4b7356966978a21a1cb59a..6f2943465bff1debc5c1fc9fcc3360218a1afbcb 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -1191,6 +1192,8 @@ static struct inode *hugetlbfs_alloc_inode(struct super_block *sb) * private inode. This simplifies hugetlbfs_destroy_inode. */ mpol_shared_policy_init(&p->policy, NULL); + /* Initialize hpool here in case of a quick call to destroy */ + link_hpool(p); return &p->vfs_inode; } @@ -1204,6 +1207,7 @@ static void hugetlbfs_destroy_inode(struct inode *inode) { hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb)); mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy); + unlink_hpool(HUGETLBFS_I(inode)); } static const struct address_space_operations hugetlbfs_aops = { diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h index 2b2c90562bcf1dd4f4b98ded167ab6e88780881a..8512f509899b0c10efa72bda03ada77c7ae66783 100644 --- a/include/linux/dynamic_hugetlb.h +++ b/include/linux/dynamic_hugetlb.h @@ -96,6 +96,11 @@ bool free_page_to_dhugetlb_pool(struct page *page); void free_page_list_to_dhugetlb_pool(struct list_head *list); int task_has_mem_in_hpool(struct task_struct *tsk); +void link_hpool(struct hugetlbfs_inode_info *p); +void unlink_hpool(struct hugetlbfs_inode_info *p); +bool file_has_mem_in_hpool(struct hugetlbfs_inode_info *p); +int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_info *p); + #else #define dhugetlb_enabled 0 @@ -134,5 +139,22 @@ static inline int task_has_mem_in_hpool(struct task_struct *tsk) return 0; } +#ifdef CONFIG_HUGETLBFS +static inline void link_hpool(struct hugetlbfs_inode_info *p) +{ +} +static inline void unlink_hpool(struct hugetlbfs_inode_info *p) +{ +} +static inline bool file_has_mem_in_hpool(struct hugetlbfs_inode_info *p) +{ + return false; +} +static inline int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_info *p) +{ + return 0; +} +#endif + #endif /* CONFIG_DYNAMIC_HUGETLB */ #endif /* __LINUX_DYNAMIC_HUGETLB_H */ diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index a1135c43719ea9c6c16a9e3f32c977c1e4efd5cc..634630ebc8a7e7d42462b9dab3aa5a61c0ce2185 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -446,6 +446,7 @@ struct hugetlbfs_inode_info { struct shared_policy policy; struct inode vfs_inode; unsigned int seals; + struct dhugetlb_pool *hpool; }; static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode) diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 423d9624d4f09a7d68e55ba443a3d2384266f52f..f8ae9ba90bcb15b2069e808d4bdb80027c623d86 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -556,6 +556,62 @@ void free_page_list_to_dhugetlb_pool(struct list_head *list) } } +void link_hpool(struct hugetlbfs_inode_info *p) +{ + if (!dhugetlb_enabled || !p) + return; + + p->hpool = find_hpool_by_task(current); + if (!get_hpool_unless_zero(p->hpool)) + p->hpool = NULL; +} + +void unlink_hpool(struct hugetlbfs_inode_info *p) +{ + if (!dhugetlb_enabled || !p) + return; + + put_hpool(p->hpool); + p->hpool = NULL; +} + +bool file_has_mem_in_hpool(struct hugetlbfs_inode_info *p) +{ + if (!dhugetlb_enabled || !p || !p->hpool) + return false; + return true; +} + +int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_info *p) +{ + struct dhugetlb_pool *hpool = p ? p->hpool : NULL; + struct huge_pages_pool *hpages_pool; + int ret = -ENOMEM; + + if (!dhugetlb_enabled || !hpool) + return 0; + + if (delta == 0) + return 0; + + spin_lock(&hpool->lock); + if (hstate_is_gigantic(h)) + hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_1G]; + else + hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_2M]; + if (delta > 0 && delta <= hpages_pool->free_huge_pages - hpages_pool->resv_huge_pages) { + hpages_pool->resv_huge_pages += delta; + ret = 0; + } else if (delta < 0) { + hpages_pool->resv_huge_pages -= (unsigned long)(-delta); + WARN_ON(hpages_pool->resv_huge_pages < 0); + ret = 0; + } + spin_unlock(&hpool->lock); + + return ret; +} + static int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, unsigned long nid, unsigned long nr_pages) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6049fd4a90502f24734eb721c2499949c7751d53..d26f0a7ca78059c21c1c1121e3c1f16c777090a5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -157,6 +157,10 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, if (!spool) return ret; + /* Skip subpool when hugetlb file belongs to a hugetlb_pool */ + if (file_has_mem_in_hpool(info)) + return ret; + spin_lock_irq(&spool->lock); if (spool->max_hpages != -1) { /* maximum size accounting */ @@ -203,6 +207,10 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, if (!spool) return delta; + /* Skip subpool when hugetlb file belongs to a hugetlb_pool */ + if (file_has_mem_in_hpool(info)) + return ret; + spin_lock_irqsave(&spool->lock, flags); if (spool->max_hpages != -1) /* maximum size accounting */ @@ -3899,6 +3907,9 @@ static int hugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_in { int ret = -ENOMEM; + if (file_has_mem_in_hpool(info)) + return dhugetlb_acct_memory(h, delta, info); + spin_lock_irq(&hugetlb_lock); /* * When cpuset is configured, it breaks the strict hugetlb page