hugetlb.c 17.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Generic hugetlb support.
 * (C) William Irwin, April 2004
 */
#include <linux/gfp.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/sysctl.h>
#include <linux/highmem.h>
#include <linux/nodemask.h>
D
David Gibson 已提交
13
#include <linux/pagemap.h>
14
#include <linux/mempolicy.h>
15
#include <linux/cpuset.h>
16
#include <linux/mutex.h>
17

D
David Gibson 已提交
18 19 20 21
#include <asm/page.h>
#include <asm/pgtable.h>

#include <linux/hugetlb.h>
22
#include "internal.h"
L
Linus Torvalds 已提交
23 24

const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
25
static unsigned long nr_huge_pages, free_huge_pages, reserved_huge_pages;
L
Linus Torvalds 已提交
26 27 28 29
unsigned long max_huge_pages;
static struct list_head hugepage_freelists[MAX_NUMNODES];
static unsigned int nr_huge_pages_node[MAX_NUMNODES];
static unsigned int free_huge_pages_node[MAX_NUMNODES];
30 31 32 33
/*
 * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
 */
static DEFINE_SPINLOCK(hugetlb_lock);
34

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
static void clear_huge_page(struct page *page, unsigned long addr)
{
	int i;

	might_sleep();
	for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
		cond_resched();
		clear_user_highpage(page + i, addr);
	}
}

static void copy_huge_page(struct page *dst, struct page *src,
			   unsigned long addr)
{
	int i;

	might_sleep();
	for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
		cond_resched();
		copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE);
	}
}

L
Linus Torvalds 已提交
58 59 60 61 62 63 64 65
static void enqueue_huge_page(struct page *page)
{
	int nid = page_to_nid(page);
	list_add(&page->lru, &hugepage_freelists[nid]);
	free_huge_pages++;
	free_huge_pages_node[nid]++;
}

66 67
static struct page *dequeue_huge_page(struct vm_area_struct *vma,
				unsigned long address)
L
Linus Torvalds 已提交
68 69 70
{
	int nid = numa_node_id();
	struct page *page = NULL;
71
	struct zonelist *zonelist = huge_zonelist(vma, address);
72
	struct zone **z;
L
Linus Torvalds 已提交
73

74 75
	for (z = zonelist->zones; *z; z++) {
		nid = (*z)->zone_pgdat->node_id;
76 77
		if (cpuset_zone_allowed(*z, GFP_HIGHUSER) &&
		    !list_empty(&hugepage_freelists[nid]))
78
			break;
L
Linus Torvalds 已提交
79
	}
80 81

	if (*z) {
L
Linus Torvalds 已提交
82 83 84 85 86 87 88 89 90
		page = list_entry(hugepage_freelists[nid].next,
				  struct page, lru);
		list_del(&page->lru);
		free_huge_pages--;
		free_huge_pages_node[nid]--;
	}
	return page;
}

91 92 93 94 95 96 97 98 99 100 101
static void free_huge_page(struct page *page)
{
	BUG_ON(page_count(page));

	INIT_LIST_HEAD(&page->lru);

	spin_lock(&hugetlb_lock);
	enqueue_huge_page(page);
	spin_unlock(&hugetlb_lock);
}

N
Nick Piggin 已提交
102
static int alloc_fresh_huge_page(void)
L
Linus Torvalds 已提交
103 104 105 106 107 108 109
{
	static int nid = 0;
	struct page *page;
	page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
					HUGETLB_PAGE_ORDER);
	nid = (nid + 1) % num_online_nodes();
	if (page) {
N
Nick Piggin 已提交
110
		page[1].lru.next = (void *)free_huge_page;	/* dtor */
111
		spin_lock(&hugetlb_lock);
L
Linus Torvalds 已提交
112 113
		nr_huge_pages++;
		nr_huge_pages_node[page_to_nid(page)]++;
114
		spin_unlock(&hugetlb_lock);
N
Nick Piggin 已提交
115 116
		put_page(page); /* free it into the hugepage allocator */
		return 1;
L
Linus Torvalds 已提交
117
	}
N
Nick Piggin 已提交
118
	return 0;
L
Linus Torvalds 已提交
119 120
}

121 122
static struct page *alloc_huge_page(struct vm_area_struct *vma,
				    unsigned long addr)
L
Linus Torvalds 已提交
123
{
124
	struct inode *inode = vma->vm_file->f_dentry->d_inode;
L
Linus Torvalds 已提交
125
	struct page *page;
126 127
	int use_reserve = 0;
	unsigned long idx;
L
Linus Torvalds 已提交
128 129

	spin_lock(&hugetlb_lock);
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147

	if (vma->vm_flags & VM_MAYSHARE) {

		/* idx = radix tree index, i.e. offset into file in
		 * HPAGE_SIZE units */
		idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)
			+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));

		/* The hugetlbfs specific inode info stores the number
		 * of "guaranteed available" (huge) pages.  That is,
		 * the first 'prereserved_hpages' pages of the inode
		 * are either already instantiated, or have been
		 * pre-reserved (by hugetlb_reserve_for_inode()). Here
		 * we're in the process of instantiating the page, so
		 * we use this to determine whether to draw from the
		 * pre-reserved pool or the truly free pool. */
		if (idx < HUGETLBFS_I(inode)->prereserved_hpages)
			use_reserve = 1;
L
Linus Torvalds 已提交
148
	}
149 150 151 152 153 154 155 156 157 158 159 160 161

	if (!use_reserve) {
		if (free_huge_pages <= reserved_huge_pages)
			goto fail;
	} else {
		BUG_ON(reserved_huge_pages == 0);
		reserved_huge_pages--;
	}

	page = dequeue_huge_page(vma, addr);
	if (!page)
		goto fail;

L
Linus Torvalds 已提交
162
	spin_unlock(&hugetlb_lock);
163
	set_page_refcounted(page);
L
Linus Torvalds 已提交
164
	return page;
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253

 fail:
	WARN_ON(use_reserve); /* reserved allocations shouldn't fail */
	spin_unlock(&hugetlb_lock);
	return NULL;
}

/* hugetlb_extend_reservation()
 *
 * Ensure that at least 'atleast' hugepages are, and will remain,
 * available to instantiate the first 'atleast' pages of the given
 * inode.  If the inode doesn't already have this many pages reserved
 * or instantiated, set aside some hugepages in the reserved pool to
 * satisfy later faults (or fail now if there aren't enough, rather
 * than getting the SIGBUS later).
 */
int hugetlb_extend_reservation(struct hugetlbfs_inode_info *info,
			       unsigned long atleast)
{
	struct inode *inode = &info->vfs_inode;
	unsigned long change_in_reserve = 0;
	int ret = 0;

	spin_lock(&hugetlb_lock);
	read_lock_irq(&inode->i_mapping->tree_lock);

	if (info->prereserved_hpages >= atleast)
		goto out;

	/* Because we always call this on shared mappings, none of the
	 * pages beyond info->prereserved_hpages can have been
	 * instantiated, so we need to reserve all of them now. */
	change_in_reserve = atleast - info->prereserved_hpages;

	if ((reserved_huge_pages + change_in_reserve) > free_huge_pages) {
		ret = -ENOMEM;
		goto out;
	}

	reserved_huge_pages += change_in_reserve;
	info->prereserved_hpages = atleast;

 out:
	read_unlock_irq(&inode->i_mapping->tree_lock);
	spin_unlock(&hugetlb_lock);

	return ret;
}

/* hugetlb_truncate_reservation()
 *
 * This returns pages reserved for the given inode to the general free
 * hugepage pool.  If the inode has any pages prereserved, but not
 * instantiated, beyond offset (atmost << HPAGE_SIZE), then release
 * them.
 */
void hugetlb_truncate_reservation(struct hugetlbfs_inode_info *info,
				  unsigned long atmost)
{
	struct inode *inode = &info->vfs_inode;
	struct address_space *mapping = inode->i_mapping;
	unsigned long idx;
	unsigned long change_in_reserve = 0;
	struct page *page;

	spin_lock(&hugetlb_lock);
	read_lock_irq(&inode->i_mapping->tree_lock);

	if (info->prereserved_hpages <= atmost)
		goto out;

	/* Count pages which were reserved, but not instantiated, and
	 * which we can now release. */
	for (idx = atmost; idx < info->prereserved_hpages; idx++) {
		page = radix_tree_lookup(&mapping->page_tree, idx);
		if (!page)
			/* Pages which are already instantiated can't
			 * be unreserved (and in fact have already
			 * been removed from the reserved pool) */
			change_in_reserve++;
	}

	BUG_ON(reserved_huge_pages < change_in_reserve);
	reserved_huge_pages -= change_in_reserve;
	info->prereserved_hpages = atmost;

 out:
	read_unlock_irq(&inode->i_mapping->tree_lock);
	spin_unlock(&hugetlb_lock);
L
Linus Torvalds 已提交
254 255 256 257 258 259
}

static int __init hugetlb_init(void)
{
	unsigned long i;

260 261 262
	if (HPAGE_SHIFT == 0)
		return 0;

L
Linus Torvalds 已提交
263 264 265 266
	for (i = 0; i < MAX_NUMNODES; ++i)
		INIT_LIST_HEAD(&hugepage_freelists[i]);

	for (i = 0; i < max_huge_pages; ++i) {
N
Nick Piggin 已提交
267
		if (!alloc_fresh_huge_page())
L
Linus Torvalds 已提交
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
			break;
	}
	max_huge_pages = free_huge_pages = nr_huge_pages = i;
	printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
	return 0;
}
module_init(hugetlb_init);

static int __init hugetlb_setup(char *s)
{
	if (sscanf(s, "%lu", &max_huge_pages) <= 0)
		max_huge_pages = 0;
	return 1;
}
__setup("hugepages=", hugetlb_setup);

#ifdef CONFIG_SYSCTL
static void update_and_free_page(struct page *page)
{
	int i;
	nr_huge_pages--;
	nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
	for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
		page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
				1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
				1 << PG_private | 1<< PG_writeback);
	}
N
Nick Piggin 已提交
295
	page[1].lru.next = NULL;
296
	set_page_refcounted(page);
L
Linus Torvalds 已提交
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
	__free_pages(page, HUGETLB_PAGE_ORDER);
}

#ifdef CONFIG_HIGHMEM
static void try_to_free_low(unsigned long count)
{
	int i, nid;
	for (i = 0; i < MAX_NUMNODES; ++i) {
		struct page *page, *next;
		list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
			if (PageHighMem(page))
				continue;
			list_del(&page->lru);
			update_and_free_page(page);
			nid = page_zone(page)->zone_pgdat->node_id;
			free_huge_pages--;
			free_huge_pages_node[nid]--;
			if (count >= nr_huge_pages)
				return;
		}
	}
}
#else
static inline void try_to_free_low(unsigned long count)
{
}
#endif

static unsigned long set_max_huge_pages(unsigned long count)
{
	while (count > nr_huge_pages) {
N
Nick Piggin 已提交
328
		if (!alloc_fresh_huge_page())
L
Linus Torvalds 已提交
329 330 331 332 333 334 335 336
			return nr_huge_pages;
	}
	if (count >= nr_huge_pages)
		return nr_huge_pages;

	spin_lock(&hugetlb_lock);
	try_to_free_low(count);
	while (count < nr_huge_pages) {
337
		struct page *page = dequeue_huge_page(NULL, 0);
L
Linus Torvalds 已提交
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
		if (!page)
			break;
		update_and_free_page(page);
	}
	spin_unlock(&hugetlb_lock);
	return nr_huge_pages;
}

int hugetlb_sysctl_handler(struct ctl_table *table, int write,
			   struct file *file, void __user *buffer,
			   size_t *length, loff_t *ppos)
{
	proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
	max_huge_pages = set_max_huge_pages(max_huge_pages);
	return 0;
}
#endif /* CONFIG_SYSCTL */

int hugetlb_report_meminfo(char *buf)
{
	return sprintf(buf,
			"HugePages_Total: %5lu\n"
			"HugePages_Free:  %5lu\n"
361
		        "HugePages_Rsvd:  %5lu\n"
L
Linus Torvalds 已提交
362 363 364
			"Hugepagesize:    %5lu kB\n",
			nr_huge_pages,
			free_huge_pages,
365
		        reserved_huge_pages,
L
Linus Torvalds 已提交
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
			HPAGE_SIZE/1024);
}

int hugetlb_report_node_meminfo(int nid, char *buf)
{
	return sprintf(buf,
		"Node %d HugePages_Total: %5u\n"
		"Node %d HugePages_Free:  %5u\n",
		nid, nr_huge_pages_node[nid],
		nid, free_huge_pages_node[nid]);
}

/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
unsigned long hugetlb_total_pages(void)
{
	return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
}

/*
 * We cannot handle pagefaults against hugetlb pages at all.  They cause
 * handle_mm_fault() to try to instantiate regular-sized pages in the
 * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
 * this far.
 */
static struct page *hugetlb_nopage(struct vm_area_struct *vma,
				unsigned long address, int *unused)
{
	BUG();
	return NULL;
}

struct vm_operations_struct hugetlb_vm_ops = {
	.nopage = hugetlb_nopage,
};

401 402
static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
				int writable)
D
David Gibson 已提交
403 404 405
{
	pte_t entry;

406
	if (writable) {
D
David Gibson 已提交
407 408 409 410 411 412 413 414 415 416 417
		entry =
		    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
	} else {
		entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
	}
	entry = pte_mkyoung(entry);
	entry = pte_mkhuge(entry);

	return entry;
}

418 419 420 421 422 423 424 425 426 427 428 429
static void set_huge_ptep_writable(struct vm_area_struct *vma,
				   unsigned long address, pte_t *ptep)
{
	pte_t entry;

	entry = pte_mkwrite(pte_mkdirty(*ptep));
	ptep_set_access_flags(vma, address, ptep, entry, 1);
	update_mmu_cache(vma, address, entry);
	lazy_mmu_prot_update(entry);
}


D
David Gibson 已提交
430 431 432 433 434
int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
			    struct vm_area_struct *vma)
{
	pte_t *src_pte, *dst_pte, entry;
	struct page *ptepage;
435
	unsigned long addr;
436 437 438
	int cow;

	cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
D
David Gibson 已提交
439

440
	for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
H
Hugh Dickins 已提交
441 442 443
		src_pte = huge_pte_offset(src, addr);
		if (!src_pte)
			continue;
D
David Gibson 已提交
444 445 446
		dst_pte = huge_pte_alloc(dst, addr);
		if (!dst_pte)
			goto nomem;
H
Hugh Dickins 已提交
447
		spin_lock(&dst->page_table_lock);
448
		spin_lock(&src->page_table_lock);
H
Hugh Dickins 已提交
449
		if (!pte_none(*src_pte)) {
450 451
			if (cow)
				ptep_set_wrprotect(src, addr, src_pte);
452 453 454
			entry = *src_pte;
			ptepage = pte_page(entry);
			get_page(ptepage);
455
			add_mm_counter(dst, file_rss, HPAGE_SIZE / PAGE_SIZE);
456 457 458
			set_huge_pte_at(dst, addr, dst_pte, entry);
		}
		spin_unlock(&src->page_table_lock);
H
Hugh Dickins 已提交
459
		spin_unlock(&dst->page_table_lock);
D
David Gibson 已提交
460 461 462 463 464 465 466 467 468 469 470 471
	}
	return 0;

nomem:
	return -ENOMEM;
}

void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
			  unsigned long end)
{
	struct mm_struct *mm = vma->vm_mm;
	unsigned long address;
472
	pte_t *ptep;
D
David Gibson 已提交
473 474 475 476 477 478 479
	pte_t pte;
	struct page *page;

	WARN_ON(!is_vm_hugetlb_page(vma));
	BUG_ON(start & ~HPAGE_MASK);
	BUG_ON(end & ~HPAGE_MASK);

480 481
	spin_lock(&mm->page_table_lock);

482 483 484
	/* Update high watermark before we lower rss */
	update_hiwater_rss(mm);

D
David Gibson 已提交
485
	for (address = start; address < end; address += HPAGE_SIZE) {
486
		ptep = huge_pte_offset(mm, address);
A
Adam Litke 已提交
487
		if (!ptep)
488 489 490
			continue;

		pte = huge_ptep_get_and_clear(mm, address, ptep);
D
David Gibson 已提交
491 492
		if (pte_none(pte))
			continue;
493

D
David Gibson 已提交
494 495
		page = pte_page(pte);
		put_page(page);
496
		add_mm_counter(mm, file_rss, (int) -(HPAGE_SIZE / PAGE_SIZE));
D
David Gibson 已提交
497 498
	}

L
Linus Torvalds 已提交
499
	spin_unlock(&mm->page_table_lock);
500
	flush_tlb_range(vma, start, end);
L
Linus Torvalds 已提交
501
}
D
David Gibson 已提交
502

503 504 505 506
static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
			unsigned long address, pte_t *ptep, pte_t pte)
{
	struct page *old_page, *new_page;
507
	int avoidcopy;
508 509 510 511 512 513 514 515 516 517 518 519

	old_page = pte_page(pte);

	/* If no-one else is actually using this page, avoid the copy
	 * and just make the page writable */
	avoidcopy = (page_count(old_page) == 1);
	if (avoidcopy) {
		set_huge_ptep_writable(vma, address, ptep);
		return VM_FAULT_MINOR;
	}

	page_cache_get(old_page);
520
	new_page = alloc_huge_page(vma, address);
521 522 523

	if (!new_page) {
		page_cache_release(old_page);
524
		return VM_FAULT_OOM;
525 526 527
	}

	spin_unlock(&mm->page_table_lock);
528
	copy_huge_page(new_page, old_page, address);
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
	spin_lock(&mm->page_table_lock);

	ptep = huge_pte_offset(mm, address & HPAGE_MASK);
	if (likely(pte_same(*ptep, pte))) {
		/* Break COW */
		set_huge_pte_at(mm, address, ptep,
				make_huge_pte(vma, new_page, 1));
		/* Make the old page be freed below */
		new_page = old_page;
	}
	page_cache_release(new_page);
	page_cache_release(old_page);
	return VM_FAULT_MINOR;
}

544
int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
545
			unsigned long address, pte_t *ptep, int write_access)
546 547
{
	int ret = VM_FAULT_SIGBUS;
A
Adam Litke 已提交
548 549 550 551
	unsigned long idx;
	unsigned long size;
	struct page *page;
	struct address_space *mapping;
552
	pte_t new_pte;
A
Adam Litke 已提交
553 554 555 556 557 558 559 560 561

	mapping = vma->vm_file->f_mapping;
	idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
		+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));

	/*
	 * Use page lock to guard against racing truncation
	 * before we get page_table_lock.
	 */
562 563 564 565 566 567 568 569
retry:
	page = find_lock_page(mapping, idx);
	if (!page) {
		if (hugetlb_get_quota(mapping))
			goto out;
		page = alloc_huge_page(vma, address);
		if (!page) {
			hugetlb_put_quota(mapping);
570
			ret = VM_FAULT_OOM;
571 572
			goto out;
		}
573
		clear_huge_page(page, address);
574

575 576 577 578 579 580 581 582 583 584 585 586 587 588
		if (vma->vm_flags & VM_SHARED) {
			int err;

			err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
			if (err) {
				put_page(page);
				hugetlb_put_quota(mapping);
				if (err == -EEXIST)
					goto retry;
				goto out;
			}
		} else
			lock_page(page);
	}
589

590
	spin_lock(&mm->page_table_lock);
A
Adam Litke 已提交
591 592 593 594 595
	size = i_size_read(mapping->host) >> HPAGE_SHIFT;
	if (idx >= size)
		goto backout;

	ret = VM_FAULT_MINOR;
596
	if (!pte_none(*ptep))
A
Adam Litke 已提交
597 598 599
		goto backout;

	add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
600 601 602 603 604 605 606 607 608
	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
				&& (vma->vm_flags & VM_SHARED)));
	set_huge_pte_at(mm, address, ptep, new_pte);

	if (write_access && !(vma->vm_flags & VM_SHARED)) {
		/* Optimization, do the COW without a second fault */
		ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
	}

609
	spin_unlock(&mm->page_table_lock);
A
Adam Litke 已提交
610 611
	unlock_page(page);
out:
612
	return ret;
A
Adam Litke 已提交
613 614 615 616 617 618 619

backout:
	spin_unlock(&mm->page_table_lock);
	hugetlb_put_quota(mapping);
	unlock_page(page);
	put_page(page);
	goto out;
620 621
}

622 623 624 625 626
int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
			unsigned long address, int write_access)
{
	pte_t *ptep;
	pte_t entry;
627
	int ret;
628
	static DEFINE_MUTEX(hugetlb_instantiation_mutex);
629 630 631 632 633

	ptep = huge_pte_alloc(mm, address);
	if (!ptep)
		return VM_FAULT_OOM;

634 635 636 637 638 639
	/*
	 * Serialize hugepage allocation and instantiation, so that we don't
	 * get spurious allocation failures if two CPUs race to instantiate
	 * the same page in the page cache.
	 */
	mutex_lock(&hugetlb_instantiation_mutex);
640
	entry = *ptep;
641 642 643 644 645
	if (pte_none(entry)) {
		ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
		mutex_unlock(&hugetlb_instantiation_mutex);
		return ret;
	}
646

647 648 649 650 651 652 653 654
	ret = VM_FAULT_MINOR;

	spin_lock(&mm->page_table_lock);
	/* Check for a racing update before calling hugetlb_cow */
	if (likely(pte_same(entry, *ptep)))
		if (write_access && !pte_write(entry))
			ret = hugetlb_cow(mm, vma, address, ptep, entry);
	spin_unlock(&mm->page_table_lock);
655
	mutex_unlock(&hugetlb_instantiation_mutex);
656 657

	return ret;
658 659
}

D
David Gibson 已提交
660 661 662 663 664 665 666 667
int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
			struct page **pages, struct vm_area_struct **vmas,
			unsigned long *position, int *length, int i)
{
	unsigned long vpfn, vaddr = *position;
	int remainder = *length;

	vpfn = vaddr/PAGE_SIZE;
668
	spin_lock(&mm->page_table_lock);
D
David Gibson 已提交
669
	while (vaddr < vma->vm_end && remainder) {
A
Adam Litke 已提交
670 671
		pte_t *pte;
		struct page *page;
D
David Gibson 已提交
672

A
Adam Litke 已提交
673 674 675 676 677 678
		/*
		 * Some archs (sparc64, sh*) have multiple pte_ts to
		 * each hugepage.  We have to make * sure we get the
		 * first, for the page indexing below to work.
		 */
		pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
D
David Gibson 已提交
679

A
Adam Litke 已提交
680 681
		if (!pte || pte_none(*pte)) {
			int ret;
D
David Gibson 已提交
682

A
Adam Litke 已提交
683 684 685 686 687
			spin_unlock(&mm->page_table_lock);
			ret = hugetlb_fault(mm, vma, vaddr, 0);
			spin_lock(&mm->page_table_lock);
			if (ret == VM_FAULT_MINOR)
				continue;
D
David Gibson 已提交
688

A
Adam Litke 已提交
689 690 691 692 693 694 695 696
			remainder = 0;
			if (!i)
				i = -EFAULT;
			break;
		}

		if (pages) {
			page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
D
David Gibson 已提交
697 698 699 700 701 702 703 704 705 706 707 708
			get_page(page);
			pages[i] = page;
		}

		if (vmas)
			vmas[i] = vma;

		vaddr += PAGE_SIZE;
		++vpfn;
		--remainder;
		++i;
	}
709
	spin_unlock(&mm->page_table_lock);
D
David Gibson 已提交
710 711 712 713 714
	*length = remainder;
	*position = vaddr;

	return i;
}
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743

void hugetlb_change_protection(struct vm_area_struct *vma,
		unsigned long address, unsigned long end, pgprot_t newprot)
{
	struct mm_struct *mm = vma->vm_mm;
	unsigned long start = address;
	pte_t *ptep;
	pte_t pte;

	BUG_ON(address >= end);
	flush_cache_range(vma, address, end);

	spin_lock(&mm->page_table_lock);
	for (; address < end; address += HPAGE_SIZE) {
		ptep = huge_pte_offset(mm, address);
		if (!ptep)
			continue;
		if (!pte_none(*ptep)) {
			pte = huge_ptep_get_and_clear(mm, address, ptep);
			pte = pte_mkhuge(pte_modify(pte, newprot));
			set_huge_pte_at(mm, address, ptep, pte);
			lazy_mmu_prot_update(pte);
		}
	}
	spin_unlock(&mm->page_table_lock);

	flush_tlb_range(vma, start, end);
}