diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 7a81e866e7b1b5cc5ac284a4fb08ffabf410e02b..8452316439870c5ddf0d5e9f158e54b2205f685d 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -1180,13 +1180,25 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, pgdir = mm->pgd; if (pgdir == NULL) return; + + /* Get VSID */ + ssize = user_segment_size(ea); + vsid = get_vsid(mm->context.id, ea, ssize); + if (!vsid) + return; + /* + * Hash doesn't like irqs. Walking linux page table with irq disabled + * saves us from holding multiple locks. + */ + local_irq_save(flags); + /* * THP pages use update_mmu_cache_pmd. We don't do * hash preload there. Hence can ignore THP here */ ptep = find_linux_pte_or_hugepte(pgdir, ea, &hugepage_shift); if (!ptep) - return; + goto out_exit; WARN_ON(hugepage_shift); #ifdef CONFIG_PPC_64K_PAGES @@ -1197,18 +1209,9 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, * page size demotion here */ if (pte_val(*ptep) & (_PAGE_4K_PFN | _PAGE_NO_CACHE)) - return; + goto out_exit; #endif /* CONFIG_PPC_64K_PAGES */ - /* Get VSID */ - ssize = user_segment_size(ea); - vsid = get_vsid(mm->context.id, ea, ssize); - if (!vsid) - return; - - /* Hash doesn't like irqs */ - local_irq_save(flags); - /* Is that local to this CPU ? */ if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) local = 1; @@ -1230,7 +1233,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, mm->context.user_psize, mm->context.user_psize, pte_val(*ptep)); - +out_exit: local_irq_restore(flags); } diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c index 3c22fa307b9b293c0475068954caf26ece28cac2..34de9e0cdc3466313eea62bbdd7079df409547c8 100644 --- a/arch/powerpc/mm/hugepage-hash64.c +++ b/arch/powerpc/mm/hugepage-hash64.c @@ -37,6 +37,9 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, /* If PMD busy, retry the access */ if (unlikely(old_pmd & _PAGE_BUSY)) return 0; + /* If PMD is trans splitting retry the access */ + if (unlikely(old_pmd & _PAGE_SPLITTING)) + return 0; /* If PMD permissions don't match, take page fault */ if (unlikely(access & ~old_pmd)) return 1; diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 8add58061003e03ddcb3af42dda619ba415b3440..e9e6882231daf8fac6eb00c3b4b3e88714224923 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -925,12 +925,16 @@ void flush_dcache_icache_hugepage(struct page *page) * (2) pointer to next table, as normal; bottom 6 bits == 0 * (3) leaf pte for huge page, bottom two bits != 00 * (4) hugepd pointer, bottom two bits == 00, next 4 bits indicate size of table + * + * So long as we atomically load page table pointers we are safe against teardown, + * we can follow the address down to the the page and take a ref on it. */ + pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift) { - pgd_t *pg; - pud_t *pu; - pmd_t *pm; + pgd_t pgd, *pgdp; + pud_t pud, *pudp; + pmd_t pmd, *pmdp; pte_t *ret_pte; hugepd_t *hpdp = NULL; unsigned pdshift = PGDIR_SHIFT; @@ -938,34 +942,42 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift if (shift) *shift = 0; - pg = pgdir + pgd_index(ea); - + pgdp = pgdir + pgd_index(ea); + pgd = ACCESS_ONCE(*pgdp); /* - * we should first check for none. That takes care of a - * a parallel hugetlb or THP pagefault moving none entries - * to respective types. + * Always operate on the local stack value. This make sure the + * value don't get updated by a parallel THP split/collapse, + * page fault or a page unmap. The return pte_t * is still not + * stable. So should be checked there for above conditions. */ - if (pgd_none(*pg)) + if (pgd_none(pgd)) return NULL; - else if (pgd_huge(*pg)) { - ret_pte = (pte_t *) pg; + else if (pgd_huge(pgd)) { + ret_pte = (pte_t *) pgdp; goto out; - } else if (is_hugepd(pg)) - hpdp = (hugepd_t *)pg; + } else if (is_hugepd(&pgd)) + hpdp = (hugepd_t *)&pgd; else { + /* + * Even if we end up with an unmap, the pgtable will not + * be freed, because we do an rcu free and here we are + * irq disabled + */ pdshift = PUD_SHIFT; - pu = pud_offset(pg, ea); + pudp = pud_offset(&pgd, ea); + pud = ACCESS_ONCE(*pudp); - if (pud_none(*pu)) + if (pud_none(pud)) return NULL; - else if (pud_huge(*pu)) { - ret_pte = (pte_t *) pu; + else if (pud_huge(pud)) { + ret_pte = (pte_t *) pudp; goto out; - } else if (is_hugepd(pu)) - hpdp = (hugepd_t *)pu; + } else if (is_hugepd(&pud)) + hpdp = (hugepd_t *)&pud; else { pdshift = PMD_SHIFT; - pm = pmd_offset(pu, ea); + pmdp = pmd_offset(&pud, ea); + pmd = ACCESS_ONCE(*pmdp); /* * A hugepage collapse is captured by pmd_none, because * it mark the pmd none and do a hpte invalidate. @@ -975,16 +987,16 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift * hpte invalidate * */ - if (pmd_none(*pm) || pmd_trans_splitting(*pm)) + if (pmd_none(pmd) || pmd_trans_splitting(pmd)) return NULL; - if (pmd_huge(*pm) || pmd_large(*pm)) { - ret_pte = (pte_t *) pm; + if (pmd_huge(pmd) || pmd_large(pmd)) { + ret_pte = (pte_t *) pmdp; goto out; - } else if (is_hugepd(pm)) - hpdp = (hugepd_t *)pm; + } else if (is_hugepd(&pmd)) + hpdp = (hugepd_t *)&pmd; else - return pte_offset_kernel(pm, ea); + return pte_offset_kernel(&pmd, ea); } } if (!hpdp) @@ -1020,6 +1032,14 @@ int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, if ((pte_val(pte) & mask) != mask) return 0; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + /* + * check for splitting here + */ + if (pmd_trans_splitting(pte_pmd(pte))) + return 0; +#endif + /* hugepages are never "special" */ VM_BUG_ON(!pfn_valid(pte_pfn(pte))); diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 0988a26e04131e5408f8f1f3a16627b21699ad4a..ccd49f9503a95104dff61ccbec25f32082e0c82e 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -508,6 +508,10 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { #ifdef CONFIG_PPC_STD_MMU + /* + * We don't need to worry about _PAGE_PRESENT here because we are + * called with either mm->page_table_lock held or ptl lock held + */ unsigned long access = 0, trap; /* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */