diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3afabc81551cfb16ca4e2aae52483c0d4adcb67c..85610dd3ac0cc157a3eb0bf10460d67e2d1b689c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2022,6 +2022,18 @@ config ASCEND_OOM 0: disable oom killer 1: enable oom killer (default,compatible with mainline) +config ASCEND_CHARGE_MIGRATE_HUGEPAGES + bool "Enable support for migrate hugepages" + depends on HUGETLBFS + default y + help + When reserved hugepages are used up, we attempts to apply for migrate + hugepages. We expect that the migrated hugepages that are applied for + can be charged in memcg to limit the memory usage. + + This option enable the feature to charge migrate hugepages to memory + cgroup. + endif endmenu diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index fd9635a6a92ffcaf04590a6398b8b0ac4ec1ba46..397e6bfa8268af3f0312963b17cbb1dfd625e12b 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -607,6 +607,45 @@ struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, int huge_add_to_page_cache(struct page *page, struct address_space *mapping, pgoff_t idx); +#ifdef CONFIG_ASCEND_FEATURES +#define HUGETLB_ALLOC_NONE 0x00 +#define HUGETLB_ALLOC_NORMAL 0x01 /* normal hugepage */ +#define HUGETLB_ALLOC_BUDDY 0x02 /* buddy hugepage */ +#define HUGETLB_ALLOC_MASK (HUGETLB_ALLOC_NONE | \ + HUGETLB_ALLOC_NORMAL | \ + HUGETLB_ALLOC_BUDDY) + +const struct hstate *hugetlb_get_hstate(void); +struct page *hugetlb_alloc_hugepage(int nid, int flag); +int hugetlb_insert_hugepage_pte(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, struct page *hpage); +int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr); +#else +static inline const struct hstate *hugetlb_get_hstate(void) +{ + return NULL; +} + +static inline struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + return NULL; +} + +static inline int hugetlb_insert_hugepage_pte(struct mm_struct *mm, + unsigned long addr, pgprot_t prot, struct page *hpage) +{ + return -EPERM; +} +static inline int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr) +{ + return -EPERM; +} +#endif + /* arch callback */ int __init __alloc_bootmem_huge_page(struct hstate *h); int __init alloc_bootmem_huge_page(struct hstate *h); @@ -1028,6 +1067,29 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr pte_t *ptep, pte_t pte, unsigned long sz) { } + +static inline const struct hstate *hugetlb_get_hstate(void) +{ + return NULL; +} + +static inline struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + return NULL; +} + +static inline int hugetlb_insert_hugepage_pte(struct mm_struct *mm, + unsigned long addr, pgprot_t prot, struct page *hpage) +{ + return -EPERM; +} + +static inline int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr) +{ + return -EPERM; +} #endif /* CONFIG_HUGETLB_PAGE */ #ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d0672e4828794be4f098b70c953a61a54e96dced..eaddb18d58e1ceade78f296f30716532de7e197f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -67,7 +67,6 @@ static struct hstate * __initdata parsed_hstate; static unsigned long __initdata default_hstate_max_huge_pages; static bool __initdata parsed_valid_hugepagesz = true; static bool __initdata parsed_default_hugepagesz; - /* * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, * free_huge_pages, and surplus_huge_pages. @@ -5907,3 +5906,141 @@ void __init hugetlb_cma_check(void) } #endif /* CONFIG_CMA */ + +#ifdef CONFIG_ASCEND_FEATURES +static int enable_charge_mighp __read_mostly; + +const struct hstate *hugetlb_get_hstate(void) +{ + return &default_hstate; +} +EXPORT_SYMBOL_GPL(hugetlb_get_hstate); + +static struct page *hugetlb_alloc_hugepage_normal(struct hstate *h, + gfp_t gfp_mask, int nid) +{ + struct page *page = NULL; + + spin_lock(&hugetlb_lock); + if (h->free_huge_pages - h->resv_huge_pages > 0) + page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL, NULL); + spin_unlock(&hugetlb_lock); + + return page; +} + +/* + * Allocate hugepage without reserve + */ +struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + struct hstate *h = &default_hstate; + gfp_t gfp_mask = htlb_alloc_mask(h); + struct page *page = NULL; + + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + if (nid < 0 || nid >= MAX_NUMNODES) + return NULL; + + if (flag & ~HUGETLB_ALLOC_MASK) + return NULL; + + gfp_mask |= __GFP_THISNODE; + if (enable_charge_mighp) + gfp_mask |= __GFP_ACCOUNT; + + if (flag & HUGETLB_ALLOC_NORMAL) + page = hugetlb_alloc_hugepage_normal(h, gfp_mask, nid); + else if (flag & HUGETLB_ALLOC_BUDDY) + page = alloc_migrate_huge_page(h, gfp_mask, nid, NULL); + else + page = alloc_huge_page_nodemask(h, nid, NULL, gfp_mask); + + return page; +} +EXPORT_SYMBOL_GPL(hugetlb_alloc_hugepage); + +static int __hugetlb_insert_hugepage(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, unsigned long pfn, bool special) +{ + int ret = 0; + pte_t *ptep, entry; + struct hstate *h; + struct vm_area_struct *vma; + struct address_space *mapping; + spinlock_t *ptl; + + h = size_to_hstate(PMD_SIZE); + if (!h) + return -EINVAL; + + if (!IS_ALIGNED(addr, PMD_SIZE)) + return -EINVAL; + + vma = find_vma(mm, addr); + if (!vma || !range_in_vma(vma, addr, addr + PMD_SIZE)) + return -EINVAL; + + mapping = vma->vm_file->f_mapping; + i_mmap_lock_read(mapping); + ptep = huge_pte_alloc(mm, addr, huge_page_size(h)); + if (!ptep) { + ret = -ENXIO; + goto out_unlock; + } + + if (WARN_ON(ptep && !pte_none(*ptep) && !pmd_huge(*(pmd_t *)ptep))) { + ret = -ENXIO; + goto out_unlock; + } + + entry = pfn_pte(pfn, prot); + entry = huge_pte_mkdirty(entry); + if (!(pgprot_val(prot) & PTE_RDONLY)) + entry = huge_pte_mkwrite(entry); + entry = pte_mkyoung(entry); + entry = pte_mkhuge(entry); + if (special) + entry = pte_mkspecial(entry); + + ptl = huge_pte_lockptr(h, mm, ptep); + spin_lock(ptl); + set_huge_pte_at(mm, addr, ptep, entry); + spin_unlock(ptl); + +out_unlock: + i_mmap_unlock_read(mapping); + + return ret; +} + +int hugetlb_insert_hugepage_pte(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, struct page *hpage) +{ + return __hugetlb_insert_hugepage(mm, addr, prot, page_to_pfn(hpage), false); +} +EXPORT_SYMBOL_GPL(hugetlb_insert_hugepage_pte); + +int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, unsigned long phy_addr) +{ + return __hugetlb_insert_hugepage(mm, addr, prot, phy_addr >> PAGE_SHIFT, true); +} +EXPORT_SYMBOL_GPL(hugetlb_insert_hugepage_pte_by_pa); + +#ifdef CONFIG_ASCEND_CHARGE_MIGRATE_HUGEPAGES + +static int __init ascend_enable_charge_migrate_hugepages(char *s) +{ + enable_charge_mighp = 1; + + pr_info("Ascend enable charge migrate hugepage\n"); + + return 1; +} +__setup("enable_charge_mighp", ascend_enable_charge_migrate_hugepages); + +#endif +#endif