inode.c 25.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * hugetlbpage-backed filesystem.  Based on ramfs.
 *
4
 * Nadia Yvette Chambers, 2002
L
Linus Torvalds 已提交
5 6 7 8
 *
 * Copyright (C) 2002 Linus Torvalds.
 */

9 10
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

L
Linus Torvalds 已提交
11 12 13 14 15 16 17
#include <linux/module.h>
#include <linux/thread_info.h>
#include <asm/current.h>
#include <linux/sched.h>		/* remove ASAP */
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/file.h>
18
#include <linux/kernel.h>
L
Linus Torvalds 已提交
19 20 21 22 23
#include <linux/writeback.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/init.h>
#include <linux/string.h>
24
#include <linux/capability.h>
25
#include <linux/ctype.h>
L
Linus Torvalds 已提交
26 27 28
#include <linux/backing-dev.h>
#include <linux/hugetlb.h>
#include <linux/pagevec.h>
29
#include <linux/parser.h>
30
#include <linux/mman.h>
L
Linus Torvalds 已提交
31 32 33 34
#include <linux/slab.h>
#include <linux/dnotify.h>
#include <linux/statfs.h>
#include <linux/security.h>
N
Nick Black 已提交
35
#include <linux/magic.h>
N
Naoya Horiguchi 已提交
36
#include <linux/migrate.h>
L
Linus Torvalds 已提交
37 38 39

#include <asm/uaccess.h>

40
static const struct super_operations hugetlbfs_ops;
41
static const struct address_space_operations hugetlbfs_aops;
42
const struct file_operations hugetlbfs_file_operations;
43 44
static const struct inode_operations hugetlbfs_dir_inode_operations;
static const struct inode_operations hugetlbfs_inode_operations;
L
Linus Torvalds 已提交
45

D
David Gibson 已提交
46
struct hugetlbfs_config {
47 48
	kuid_t   uid;
	kgid_t   gid;
D
David Gibson 已提交
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
	umode_t mode;
	long	nr_blocks;
	long	nr_inodes;
	struct hstate *hstate;
};

struct hugetlbfs_inode_info {
	struct shared_policy policy;
	struct inode vfs_inode;
};

static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
{
	return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
}

L
Linus Torvalds 已提交
65 66
int sysctl_hugetlb_shm_group;

67 68 69
enum {
	Opt_size, Opt_nr_inodes,
	Opt_mode, Opt_uid, Opt_gid,
70
	Opt_pagesize,
71 72 73
	Opt_err,
};

74
static const match_table_t tokens = {
75 76 77 78 79
	{Opt_size,	"size=%s"},
	{Opt_nr_inodes,	"nr_inodes=%s"},
	{Opt_mode,	"mode=%o"},
	{Opt_uid,	"uid=%u"},
	{Opt_gid,	"gid=%u"},
80
	{Opt_pagesize,	"pagesize=%s"},
81 82 83
	{Opt_err,	NULL},
};

84 85 86 87 88 89 90 91 92 93
static void huge_pagevec_release(struct pagevec *pvec)
{
	int i;

	for (i = 0; i < pagevec_count(pvec); ++i)
		put_page(pvec->pages[i]);

	pagevec_reinit(pvec);
}

L
Linus Torvalds 已提交
94 95
static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
{
A
Al Viro 已提交
96
	struct inode *inode = file_inode(file);
L
Linus Torvalds 已提交
97 98
	loff_t len, vma_len;
	int ret;
99
	struct hstate *h = hstate_file(file);
L
Linus Torvalds 已提交
100

101
	/*
102 103 104 105 106 107
	 * vma address alignment (but not the pgoff alignment) has
	 * already been checked by prepare_hugepage_range.  If you add
	 * any error returns here, do so after setting VM_HUGETLB, so
	 * is_vm_hugetlb_page tests below unmap_region go the right
	 * way when do_mmap_pgoff unwinds (may be important on powerpc
	 * and ia64).
108
	 */
109
	vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
110
	vma->vm_ops = &hugetlb_vm_ops;
L
Linus Torvalds 已提交
111

112
	if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
113 114
		return -EINVAL;

L
Linus Torvalds 已提交
115 116
	vma_len = (loff_t)(vma->vm_end - vma->vm_start);

117
	mutex_lock(&inode->i_mutex);
L
Linus Torvalds 已提交
118 119 120 121 122
	file_accessed(file);

	ret = -ENOMEM;
	len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);

123
	if (hugetlb_reserve_pages(inode,
124
				vma->vm_pgoff >> huge_page_order(h),
125 126
				len >> huge_page_shift(h), vma,
				vma->vm_flags))
127
		goto out;
128

A
Adam Litke 已提交
129 130
	ret = 0;
	hugetlb_prefault_arch_hook(vma->vm_mm);
131
	if (vma->vm_flags & VM_WRITE && inode->i_size < len)
L
Linus Torvalds 已提交
132 133
		inode->i_size = len;
out:
134
	mutex_unlock(&inode->i_mutex);
L
Linus Torvalds 已提交
135 136 137 138 139

	return ret;
}

/*
140
 * Called under down_write(mmap_sem).
L
Linus Torvalds 已提交
141 142
 */

143
#ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
L
Linus Torvalds 已提交
144 145 146 147 148 149
static unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
150
	struct hstate *h = hstate_file(file);
151
	struct vm_unmapped_area_info info;
L
Linus Torvalds 已提交
152

153
	if (len & ~huge_page_mask(h))
L
Linus Torvalds 已提交
154 155 156 157
		return -EINVAL;
	if (len > TASK_SIZE)
		return -ENOMEM;

158
	if (flags & MAP_FIXED) {
159
		if (prepare_hugepage_range(file, addr, len))
160 161 162 163
			return -EINVAL;
		return addr;
	}

L
Linus Torvalds 已提交
164
	if (addr) {
165
		addr = ALIGN(addr, huge_page_size(h));
L
Linus Torvalds 已提交
166 167 168 169 170 171
		vma = find_vma(mm, addr);
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vma->vm_start))
			return addr;
	}

172 173 174 175 176 177 178
	info.flags = 0;
	info.length = len;
	info.low_limit = TASK_UNMAPPED_BASE;
	info.high_limit = TASK_SIZE;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	return vm_unmapped_area(&info);
L
Linus Torvalds 已提交
179 180 181
}
#endif

B
Badari Pulavarty 已提交
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
static int
hugetlbfs_read_actor(struct page *page, unsigned long offset,
			char __user *buf, unsigned long count,
			unsigned long size)
{
	char *kaddr;
	unsigned long left, copied = 0;
	int i, chunksize;

	if (size > count)
		size = count;

	/* Find which 4k chunk and offset with in that chunk */
	i = offset >> PAGE_CACHE_SHIFT;
	offset = offset & ~PAGE_CACHE_MASK;

	while (size) {
		chunksize = PAGE_CACHE_SIZE;
		if (offset)
			chunksize -= offset;
		if (chunksize > size)
			chunksize = size;
		kaddr = kmap(&page[i]);
		left = __copy_to_user(buf, kaddr + offset, chunksize);
		kunmap(&page[i]);
		if (left) {
			copied += (chunksize - left);
			break;
		}
		offset = 0;
		size -= chunksize;
		buf += chunksize;
		copied += chunksize;
		i++;
	}
	return copied ? copied : -EFAULT;
}

/*
 * Support for read() - Find the page attached to f_mapping and copy out the
 * data. Its *very* similar to do_generic_mapping_read(), we can't use that
 * since it has PAGE_CACHE_SIZE assumptions.
 */
static ssize_t hugetlbfs_read(struct file *filp, char __user *buf,
			      size_t len, loff_t *ppos)
{
228
	struct hstate *h = hstate_file(filp);
B
Badari Pulavarty 已提交
229 230
	struct address_space *mapping = filp->f_mapping;
	struct inode *inode = mapping->host;
231 232
	unsigned long index = *ppos >> huge_page_shift(h);
	unsigned long offset = *ppos & ~huge_page_mask(h);
B
Badari Pulavarty 已提交
233 234 235 236 237 238 239 240 241 242
	unsigned long end_index;
	loff_t isize;
	ssize_t retval = 0;

	/* validate length */
	if (len == 0)
		goto out;

	for (;;) {
		struct page *page;
243
		unsigned long nr, ret;
244
		int ra;
B
Badari Pulavarty 已提交
245 246

		/* nr is the maximum number of bytes to copy from this page */
247
		nr = huge_page_size(h);
248 249 250 251
		isize = i_size_read(inode);
		if (!isize)
			goto out;
		end_index = (isize - 1) >> huge_page_shift(h);
B
Badari Pulavarty 已提交
252 253 254
		if (index >= end_index) {
			if (index > end_index)
				goto out;
255
			nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
256
			if (nr <= offset)
B
Badari Pulavarty 已提交
257 258 259 260 261
				goto out;
		}
		nr = nr - offset;

		/* Find the page */
262
		page = find_lock_page(mapping, index);
B
Badari Pulavarty 已提交
263 264 265 266 267 268 269
		if (unlikely(page == NULL)) {
			/*
			 * We have a HOLE, zero out the user-buffer for the
			 * length of the hole or request.
			 */
			ret = len < nr ? len : nr;
			if (clear_user(buf, ret))
270 271 272
				ra = -EFAULT;
			else
				ra = 0;
B
Badari Pulavarty 已提交
273
		} else {
274 275
			unlock_page(page);

B
Badari Pulavarty 已提交
276 277 278
			/*
			 * We have the page, copy it to user space buffer.
			 */
279 280
			ra = hugetlbfs_read_actor(page, offset, buf, len, nr);
			ret = ra;
281
			page_cache_release(page);
B
Badari Pulavarty 已提交
282
		}
283
		if (ra < 0) {
B
Badari Pulavarty 已提交
284
			if (retval == 0)
285
				retval = ra;
B
Badari Pulavarty 已提交
286 287 288 289 290 291
			goto out;
		}

		offset += ret;
		retval += ret;
		len -= ret;
292 293
		index += offset >> huge_page_shift(h);
		offset &= ~huge_page_mask(h);
B
Badari Pulavarty 已提交
294 295 296 297 298 299

		/* short read or no more work */
		if ((ret != nr) || (len == 0))
			break;
	}
out:
300
	*ppos = ((loff_t)index << huge_page_shift(h)) + offset;
B
Badari Pulavarty 已提交
301 302 303
	return retval;
}

N
Nick Piggin 已提交
304 305 306 307
static int hugetlbfs_write_begin(struct file *file,
			struct address_space *mapping,
			loff_t pos, unsigned len, unsigned flags,
			struct page **pagep, void **fsdata)
L
Linus Torvalds 已提交
308 309 310 311
{
	return -EINVAL;
}

N
Nick Piggin 已提交
312 313 314
static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
L
Linus Torvalds 已提交
315
{
N
Nick Piggin 已提交
316
	BUG();
L
Linus Torvalds 已提交
317 318 319 320 321
	return -EINVAL;
}

static void truncate_huge_page(struct page *page)
{
322
	cancel_dirty_page(page, /* No IO accounting for huge pages? */0);
L
Linus Torvalds 已提交
323
	ClearPageUptodate(page);
324
	delete_from_page_cache(page);
L
Linus Torvalds 已提交
325 326
}

327
static void truncate_hugepages(struct inode *inode, loff_t lstart)
L
Linus Torvalds 已提交
328
{
329
	struct hstate *h = hstate_inode(inode);
330
	struct address_space *mapping = &inode->i_data;
331
	const pgoff_t start = lstart >> huge_page_shift(h);
L
Linus Torvalds 已提交
332 333
	struct pagevec pvec;
	pgoff_t next;
334
	int i, freed = 0;
L
Linus Torvalds 已提交
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354

	pagevec_init(&pvec, 0);
	next = start;
	while (1) {
		if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
			if (next == start)
				break;
			next = start;
			continue;
		}

		for (i = 0; i < pagevec_count(&pvec); ++i) {
			struct page *page = pvec.pages[i];

			lock_page(page);
			if (page->index > next)
				next = page->index;
			++next;
			truncate_huge_page(page);
			unlock_page(page);
355
			freed++;
L
Linus Torvalds 已提交
356 357 358 359
		}
		huge_pagevec_release(&pvec);
	}
	BUG_ON(!lstart && mapping->nrpages);
360
	hugetlb_unreserve_pages(inode, start, freed);
L
Linus Torvalds 已提交
361 362
}

A
Al Viro 已提交
363
static void hugetlbfs_evict_inode(struct inode *inode)
L
Linus Torvalds 已提交
364
{
365 366
	struct resv_map *resv_map;

367
	truncate_hugepages(inode, 0);
368 369 370 371
	resv_map = (struct resv_map *)inode->i_mapping->private_data;
	/* root inode doesn't have the resv_map, so we should check it */
	if (resv_map)
		resv_map_release(&resv_map->refs);
372
	clear_inode(inode);
373 374
}

L
Linus Torvalds 已提交
375
static inline void
376
hugetlb_vmtruncate_list(struct rb_root *root, pgoff_t pgoff)
L
Linus Torvalds 已提交
377 378 379
{
	struct vm_area_struct *vma;

380
	vma_interval_tree_foreach(vma, root, pgoff, ULONG_MAX) {
L
Linus Torvalds 已提交
381 382 383
		unsigned long v_offset;

		/*
H
Hugh Dickins 已提交
384
		 * Can the expression below overflow on 32-bit arches?
385
		 * No, because the interval tree returns us only those vmas
H
Hugh Dickins 已提交
386 387
		 * which overlap the truncated area starting at pgoff,
		 * and no vma on a 32-bit arch can span beyond the 4GB.
L
Linus Torvalds 已提交
388
		 */
H
Hugh Dickins 已提交
389 390 391
		if (vma->vm_pgoff < pgoff)
			v_offset = (pgoff - vma->vm_pgoff) << PAGE_SHIFT;
		else
L
Linus Torvalds 已提交
392 393
			v_offset = 0;

394 395
		unmap_hugepage_range(vma, vma->vm_start + v_offset,
				     vma->vm_end, NULL);
L
Linus Torvalds 已提交
396 397 398 399 400
	}
}

static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
{
H
Hugh Dickins 已提交
401
	pgoff_t pgoff;
L
Linus Torvalds 已提交
402
	struct address_space *mapping = inode->i_mapping;
403
	struct hstate *h = hstate_inode(inode);
L
Linus Torvalds 已提交
404

405
	BUG_ON(offset & ~huge_page_mask(h));
H
Hugh Dickins 已提交
406
	pgoff = offset >> PAGE_SHIFT;
L
Linus Torvalds 已提交
407

408
	i_size_write(inode, offset);
409
	i_mmap_lock_write(mapping);
410
	if (!RB_EMPTY_ROOT(&mapping->i_mmap))
L
Linus Torvalds 已提交
411
		hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
412
	i_mmap_unlock_write(mapping);
413
	truncate_hugepages(inode, offset);
L
Linus Torvalds 已提交
414 415 416 417 418 419
	return 0;
}

static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
{
	struct inode *inode = dentry->d_inode;
420
	struct hstate *h = hstate_inode(inode);
L
Linus Torvalds 已提交
421 422 423 424 425 426 427
	int error;
	unsigned int ia_valid = attr->ia_valid;

	BUG_ON(!inode);

	error = inode_change_ok(inode, attr);
	if (error)
C
Christoph Hellwig 已提交
428
		return error;
L
Linus Torvalds 已提交
429 430 431

	if (ia_valid & ATTR_SIZE) {
		error = -EINVAL;
C
Christoph Hellwig 已提交
432 433 434
		if (attr->ia_size & ~huge_page_mask(h))
			return -EINVAL;
		error = hugetlb_vmtruncate(inode, attr->ia_size);
L
Linus Torvalds 已提交
435
		if (error)
C
Christoph Hellwig 已提交
436
			return error;
L
Linus Torvalds 已提交
437
	}
C
Christoph Hellwig 已提交
438 439 440 441

	setattr_copy(inode, attr);
	mark_inode_dirty(inode);
	return 0;
L
Linus Torvalds 已提交
442 443
}

444 445
static struct inode *hugetlbfs_get_root(struct super_block *sb,
					struct hugetlbfs_config *config)
L
Linus Torvalds 已提交
446 447 448 449 450 451
{
	struct inode *inode;

	inode = new_inode(sb);
	if (inode) {
		struct hugetlbfs_inode_info *info;
452
		inode->i_ino = get_next_ino();
453 454 455 456 457 458 459 460 461 462
		inode->i_mode = S_IFDIR | config->mode;
		inode->i_uid = config->uid;
		inode->i_gid = config->gid;
		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
		info = HUGETLBFS_I(inode);
		mpol_shared_policy_init(&info->policy, NULL);
		inode->i_op = &hugetlbfs_dir_inode_operations;
		inode->i_fop = &simple_dir_operations;
		/* directory inodes start off with i_nlink == 2 (for "." entry) */
		inc_nlink(inode);
463
		lockdep_annotate_inode_mutex_key(inode);
464 465 466 467
	}
	return inode;
}

468
/*
469
 * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
470 471
 * be taken from reclaim -- unlike regular filesystems. This needs an
 * annotation because huge_pmd_share() does an allocation under
472
 * i_mmap_rwsem.
473
 */
474
static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
475

476 477
static struct inode *hugetlbfs_get_inode(struct super_block *sb,
					struct inode *dir,
A
Al Viro 已提交
478
					umode_t mode, dev_t dev)
479 480
{
	struct inode *inode;
481 482 483 484 485
	struct resv_map *resv_map;

	resv_map = resv_map_alloc();
	if (!resv_map)
		return NULL;
486 487 488 489 490 491

	inode = new_inode(sb);
	if (inode) {
		struct hugetlbfs_inode_info *info;
		inode->i_ino = get_next_ino();
		inode_init_owner(inode, dir, mode);
492 493
		lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
				&hugetlbfs_i_mmap_rwsem_key);
L
Linus Torvalds 已提交
494 495
		inode->i_mapping->a_ops = &hugetlbfs_aops;
		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
496
		inode->i_mapping->private_data = resv_map;
L
Linus Torvalds 已提交
497
		info = HUGETLBFS_I(inode);
498 499 500 501 502 503 504
		/*
		 * The policy is initialized here even if we are creating a
		 * private inode because initialization simply creates an
		 * an empty rb tree and calls spin_lock_init(), later when we
		 * call mpol_free_shared_policy() it will just return because
		 * the rb tree will still be empty.
		 */
505
		mpol_shared_policy_init(&info->policy, NULL);
L
Linus Torvalds 已提交
506 507 508 509 510 511 512 513 514 515 516 517 518
		switch (mode & S_IFMT) {
		default:
			init_special_inode(inode, mode, dev);
			break;
		case S_IFREG:
			inode->i_op = &hugetlbfs_inode_operations;
			inode->i_fop = &hugetlbfs_file_operations;
			break;
		case S_IFDIR:
			inode->i_op = &hugetlbfs_dir_inode_operations;
			inode->i_fop = &simple_dir_operations;

			/* directory inodes start off with i_nlink == 2 (for "." entry) */
519
			inc_nlink(inode);
L
Linus Torvalds 已提交
520 521 522 523 524
			break;
		case S_IFLNK:
			inode->i_op = &page_symlink_inode_operations;
			break;
		}
525
		lockdep_annotate_inode_mutex_key(inode);
526 527 528
	} else
		kref_put(&resv_map->refs, resv_map_release);

L
Linus Torvalds 已提交
529 530 531 532 533 534 535
	return inode;
}

/*
 * File creation. Allocate an inode, and we're done..
 */
static int hugetlbfs_mknod(struct inode *dir,
A
Al Viro 已提交
536
			struct dentry *dentry, umode_t mode, dev_t dev)
L
Linus Torvalds 已提交
537 538 539
{
	struct inode *inode;
	int error = -ENOSPC;
540 541

	inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
L
Linus Torvalds 已提交
542 543 544 545 546 547 548 549 550
	if (inode) {
		dir->i_ctime = dir->i_mtime = CURRENT_TIME;
		d_instantiate(dentry, inode);
		dget(dentry);	/* Extra count - pin the dentry in core */
		error = 0;
	}
	return error;
}

551
static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
L
Linus Torvalds 已提交
552 553 554
{
	int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
	if (!retval)
555
		inc_nlink(dir);
L
Linus Torvalds 已提交
556 557 558
	return retval;
}

A
Al Viro 已提交
559
static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
L
Linus Torvalds 已提交
560 561 562 563 564 565 566 567 568 569
{
	return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
}

static int hugetlbfs_symlink(struct inode *dir,
			struct dentry *dentry, const char *symname)
{
	struct inode *inode;
	int error = -ENOSPC;

570
	inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
L
Linus Torvalds 已提交
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
	if (inode) {
		int l = strlen(symname)+1;
		error = page_symlink(inode, symname, l);
		if (!error) {
			d_instantiate(dentry, inode);
			dget(dentry);
		} else
			iput(inode);
	}
	dir->i_ctime = dir->i_mtime = CURRENT_TIME;

	return error;
}

/*
586
 * mark the head page dirty
L
Linus Torvalds 已提交
587 588 589
 */
static int hugetlbfs_set_page_dirty(struct page *page)
{
590
	struct page *head = compound_head(page);
591 592

	SetPageDirty(head);
L
Linus Torvalds 已提交
593 594 595
	return 0;
}

N
Naoya Horiguchi 已提交
596
static int hugetlbfs_migrate_page(struct address_space *mapping,
597
				struct page *newpage, struct page *page,
598
				enum migrate_mode mode)
N
Naoya Horiguchi 已提交
599 600 601 602
{
	int rc;

	rc = migrate_huge_page_move_mapping(mapping, newpage, page);
603
	if (rc != MIGRATEPAGE_SUCCESS)
N
Naoya Horiguchi 已提交
604 605 606
		return rc;
	migrate_page_copy(newpage, page);

607
	return MIGRATEPAGE_SUCCESS;
N
Naoya Horiguchi 已提交
608 609
}

610
static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
L
Linus Torvalds 已提交
611
{
612
	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
613
	struct hstate *h = hstate_inode(dentry->d_inode);
L
Linus Torvalds 已提交
614 615

	buf->f_type = HUGETLBFS_MAGIC;
616
	buf->f_bsize = huge_page_size(h);
L
Linus Torvalds 已提交
617 618
	if (sbinfo) {
		spin_lock(&sbinfo->stat_lock);
619 620
		/* If no limits set, just report 0 for max/free/used
		 * blocks, like simple_statfs() */
621 622 623 624 625 626 627 628 629
		if (sbinfo->spool) {
			long free_pages;

			spin_lock(&sbinfo->spool->lock);
			buf->f_blocks = sbinfo->spool->max_hpages;
			free_pages = sbinfo->spool->max_hpages
				- sbinfo->spool->used_hpages;
			buf->f_bavail = buf->f_bfree = free_pages;
			spin_unlock(&sbinfo->spool->lock);
630 631 632
			buf->f_files = sbinfo->max_inodes;
			buf->f_ffree = sbinfo->free_inodes;
		}
L
Linus Torvalds 已提交
633 634 635 636 637 638 639 640 641 642 643 644
		spin_unlock(&sbinfo->stat_lock);
	}
	buf->f_namelen = NAME_MAX;
	return 0;
}

static void hugetlbfs_put_super(struct super_block *sb)
{
	struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);

	if (sbi) {
		sb->s_fs_info = NULL;
645 646 647 648

		if (sbi->spool)
			hugepage_put_subpool(sbi->spool);

L
Linus Torvalds 已提交
649 650 651 652
		kfree(sbi);
	}
}

653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
{
	if (sbinfo->free_inodes >= 0) {
		spin_lock(&sbinfo->stat_lock);
		if (unlikely(!sbinfo->free_inodes)) {
			spin_unlock(&sbinfo->stat_lock);
			return 0;
		}
		sbinfo->free_inodes--;
		spin_unlock(&sbinfo->stat_lock);
	}

	return 1;
}

static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
{
	if (sbinfo->free_inodes >= 0) {
		spin_lock(&sbinfo->stat_lock);
		sbinfo->free_inodes++;
		spin_unlock(&sbinfo->stat_lock);
	}
}


678
static struct kmem_cache *hugetlbfs_inode_cachep;
L
Linus Torvalds 已提交
679 680 681

static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
{
682
	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
L
Linus Torvalds 已提交
683 684
	struct hugetlbfs_inode_info *p;

685 686
	if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
		return NULL;
687
	p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
688 689
	if (unlikely(!p)) {
		hugetlbfs_inc_free_inodes(sbinfo);
L
Linus Torvalds 已提交
690
		return NULL;
691
	}
L
Linus Torvalds 已提交
692 693 694
	return &p->vfs_inode;
}

N
Nick Piggin 已提交
695 696 697 698 699 700
static void hugetlbfs_i_callback(struct rcu_head *head)
{
	struct inode *inode = container_of(head, struct inode, i_rcu);
	kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
}

L
Linus Torvalds 已提交
701 702
static void hugetlbfs_destroy_inode(struct inode *inode)
{
703
	hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
L
Linus Torvalds 已提交
704
	mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
N
Nick Piggin 已提交
705
	call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
L
Linus Torvalds 已提交
706 707
}

708
static const struct address_space_operations hugetlbfs_aops = {
N
Nick Piggin 已提交
709 710
	.write_begin	= hugetlbfs_write_begin,
	.write_end	= hugetlbfs_write_end,
L
Linus Torvalds 已提交
711
	.set_page_dirty	= hugetlbfs_set_page_dirty,
N
Naoya Horiguchi 已提交
712
	.migratepage    = hugetlbfs_migrate_page,
L
Linus Torvalds 已提交
713 714
};

715

716
static void init_once(void *foo)
717 718 719
{
	struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;

C
Christoph Lameter 已提交
720
	inode_init_once(&ei->vfs_inode);
721 722
}

723
const struct file_operations hugetlbfs_file_operations = {
B
Badari Pulavarty 已提交
724
	.read			= hugetlbfs_read,
L
Linus Torvalds 已提交
725
	.mmap			= hugetlbfs_file_mmap,
726
	.fsync			= noop_fsync,
L
Linus Torvalds 已提交
727
	.get_unmapped_area	= hugetlb_get_unmapped_area,
728
	.llseek		= default_llseek,
L
Linus Torvalds 已提交
729 730
};

731
static const struct inode_operations hugetlbfs_dir_inode_operations = {
L
Linus Torvalds 已提交
732 733 734 735 736 737 738 739 740 741 742 743
	.create		= hugetlbfs_create,
	.lookup		= simple_lookup,
	.link		= simple_link,
	.unlink		= simple_unlink,
	.symlink	= hugetlbfs_symlink,
	.mkdir		= hugetlbfs_mkdir,
	.rmdir		= simple_rmdir,
	.mknod		= hugetlbfs_mknod,
	.rename		= simple_rename,
	.setattr	= hugetlbfs_setattr,
};

744
static const struct inode_operations hugetlbfs_inode_operations = {
L
Linus Torvalds 已提交
745 746 747
	.setattr	= hugetlbfs_setattr,
};

748
static const struct super_operations hugetlbfs_ops = {
L
Linus Torvalds 已提交
749 750
	.alloc_inode    = hugetlbfs_alloc_inode,
	.destroy_inode  = hugetlbfs_destroy_inode,
A
Al Viro 已提交
751
	.evict_inode	= hugetlbfs_evict_inode,
L
Linus Torvalds 已提交
752 753
	.statfs		= hugetlbfs_statfs,
	.put_super	= hugetlbfs_put_super,
M
Miklos Szeredi 已提交
754
	.show_options	= generic_show_options,
L
Linus Torvalds 已提交
755 756 757 758 759
};

static int
hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
{
760 761 762
	char *p, *rest;
	substring_t args[MAX_OPT_ARGS];
	int option;
763 764
	unsigned long long size = 0;
	enum { NO_SIZE, SIZE_STD, SIZE_PERCENT } setsize = NO_SIZE;
L
Linus Torvalds 已提交
765 766 767 768

	if (!options)
		return 0;

769 770
	while ((p = strsep(&options, ",")) != NULL) {
		int token;
771 772
		if (!*p)
			continue;
773 774 775 776 777 778

		token = match_token(p, tokens, args);
		switch (token) {
		case Opt_uid:
			if (match_int(&args[0], &option))
 				goto bad_val;
779 780 781
			pconfig->uid = make_kuid(current_user_ns(), option);
			if (!uid_valid(pconfig->uid))
				goto bad_val;
782 783 784 785 786
			break;

		case Opt_gid:
			if (match_int(&args[0], &option))
 				goto bad_val;
787 788 789
			pconfig->gid = make_kgid(current_user_ns(), option);
			if (!gid_valid(pconfig->gid))
				goto bad_val;
790 791 792 793 794
			break;

		case Opt_mode:
			if (match_octal(&args[0], &option))
 				goto bad_val;
795
			pconfig->mode = option & 01777U;
796 797 798 799 800 801 802
			break;

		case Opt_size: {
			/* memparse() will accept a K/M/G without a digit */
			if (!isdigit(*args[0].from))
				goto bad_val;
			size = memparse(args[0].from, &rest);
803 804 805
			setsize = SIZE_STD;
			if (*rest == '%')
				setsize = SIZE_PERCENT;
806 807
			break;
		}
L
Linus Torvalds 已提交
808

809 810 811 812 813 814 815
		case Opt_nr_inodes:
			/* memparse() will accept a K/M/G without a digit */
			if (!isdigit(*args[0].from))
				goto bad_val;
			pconfig->nr_inodes = memparse(args[0].from, &rest);
			break;

816 817 818 819 820
		case Opt_pagesize: {
			unsigned long ps;
			ps = memparse(args[0].from, &rest);
			pconfig->hstate = size_to_hstate(ps);
			if (!pconfig->hstate) {
821
				pr_err("Unsupported page size %lu MB\n",
822 823 824 825 826 827
					ps >> 20);
				return -EINVAL;
			}
			break;
		}

828
		default:
829
			pr_err("Bad mount option: \"%s\"\n", p);
830
			return -EINVAL;
831 832
			break;
		}
L
Linus Torvalds 已提交
833
	}
834 835 836 837 838 839 840 841 842 843 844 845

	/* Do size after hstate is set up */
	if (setsize > NO_SIZE) {
		struct hstate *h = pconfig->hstate;
		if (setsize == SIZE_PERCENT) {
			size <<= huge_page_shift(h);
			size *= h->max_huge_pages;
			do_div(size, 100);
		}
		pconfig->nr_blocks = (size >> huge_page_shift(h));
	}

L
Linus Torvalds 已提交
846
	return 0;
847 848

bad_val:
849
	pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
850
 	return -EINVAL;
L
Linus Torvalds 已提交
851 852 853 854 855 856 857 858 859
}

static int
hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
{
	int ret;
	struct hugetlbfs_config config;
	struct hugetlbfs_sb_info *sbinfo;

M
Miklos Szeredi 已提交
860 861
	save_mount_options(sb, data);

L
Linus Torvalds 已提交
862 863
	config.nr_blocks = -1; /* No limit on size by default */
	config.nr_inodes = -1; /* No limit on number of inodes by default */
864 865
	config.uid = current_fsuid();
	config.gid = current_fsgid();
L
Linus Torvalds 已提交
866
	config.mode = 0755;
867
	config.hstate = &default_hstate;
L
Linus Torvalds 已提交
868 869 870 871 872 873 874 875
	ret = hugetlbfs_parse_options(data, &config);
	if (ret)
		return ret;

	sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
	if (!sbinfo)
		return -ENOMEM;
	sb->s_fs_info = sbinfo;
876
	sbinfo->hstate = config.hstate;
L
Linus Torvalds 已提交
877 878 879
	spin_lock_init(&sbinfo->stat_lock);
	sbinfo->max_inodes = config.nr_inodes;
	sbinfo->free_inodes = config.nr_inodes;
880 881 882 883 884 885
	sbinfo->spool = NULL;
	if (config.nr_blocks != -1) {
		sbinfo->spool = hugepage_new_subpool(config.nr_blocks);
		if (!sbinfo->spool)
			goto out_free;
	}
L
Linus Torvalds 已提交
886
	sb->s_maxbytes = MAX_LFS_FILESIZE;
887 888
	sb->s_blocksize = huge_page_size(config.hstate);
	sb->s_blocksize_bits = huge_page_shift(config.hstate);
L
Linus Torvalds 已提交
889 890 891
	sb->s_magic = HUGETLBFS_MAGIC;
	sb->s_op = &hugetlbfs_ops;
	sb->s_time_gran = 1;
892 893
	sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
	if (!sb->s_root)
L
Linus Torvalds 已提交
894 895 896
		goto out_free;
	return 0;
out_free:
897
	kfree(sbinfo->spool);
L
Linus Torvalds 已提交
898 899 900 901
	kfree(sbinfo);
	return -ENOMEM;
}

A
Al Viro 已提交
902 903
static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
	int flags, const char *dev_name, void *data)
L
Linus Torvalds 已提交
904
{
A
Al Viro 已提交
905
	return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
L
Linus Torvalds 已提交
906 907 908 909
}

static struct file_system_type hugetlbfs_fs_type = {
	.name		= "hugetlbfs",
A
Al Viro 已提交
910
	.mount		= hugetlbfs_mount,
L
Linus Torvalds 已提交
911 912
	.kill_sb	= kill_litter_super,
};
913
MODULE_ALIAS_FS("hugetlbfs");
L
Linus Torvalds 已提交
914

915
static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
L
Linus Torvalds 已提交
916

917
static int can_do_hugetlb_shm(void)
L
Linus Torvalds 已提交
918
{
919 920 921
	kgid_t shm_group;
	shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
	return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
L
Linus Torvalds 已提交
922 923
}

924 925
static int get_hstate_idx(int page_size_log)
{
926
	struct hstate *h = hstate_sizelog(page_size_log);
927 928 929 930 931 932

	if (!h)
		return -1;
	return h - hstates;
}

933
static const struct dentry_operations anon_ops = {
934
	.d_dname = simple_dname
935 936
};

937 938 939 940 941 942
/*
 * Note that size should be aligned to proper hugepage size in caller side,
 * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
 */
struct file *hugetlb_file_setup(const char *name, size_t size,
				vm_flags_t acctflag, struct user_struct **user,
943
				int creat_flags, int page_size_log)
L
Linus Torvalds 已提交
944
{
945
	struct file *file = ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
946
	struct inode *inode;
947
	struct path path;
948
	struct super_block *sb;
L
Linus Torvalds 已提交
949
	struct qstr quick_string;
950 951 952 953 954
	int hstate_idx;

	hstate_idx = get_hstate_idx(page_size_log);
	if (hstate_idx < 0)
		return ERR_PTR(-ENODEV);
L
Linus Torvalds 已提交
955

956
	*user = NULL;
957
	if (!hugetlbfs_vfsmount[hstate_idx])
958 959
		return ERR_PTR(-ENOENT);

960
	if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
961 962
		*user = current_user();
		if (user_shm_lock(size, *user)) {
963
			task_lock(current);
964
			pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
965 966
				current->comm, current->pid);
			task_unlock(current);
967 968
		} else {
			*user = NULL;
969
			return ERR_PTR(-EPERM);
970
		}
971
	}
L
Linus Torvalds 已提交
972

973
	sb = hugetlbfs_vfsmount[hstate_idx]->mnt_sb;
974
	quick_string.name = name;
L
Linus Torvalds 已提交
975 976
	quick_string.len = strlen(quick_string.name);
	quick_string.hash = 0;
977
	path.dentry = d_alloc_pseudo(sb, &quick_string);
978
	if (!path.dentry)
L
Linus Torvalds 已提交
979 980
		goto out_shm_unlock;

981
	d_set_d_op(path.dentry, &anon_ops);
982
	path.mnt = mntget(hugetlbfs_vfsmount[hstate_idx]);
983
	file = ERR_PTR(-ENOSPC);
984
	inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);
L
Linus Torvalds 已提交
985
	if (!inode)
986
		goto out_dentry;
L
Linus Torvalds 已提交
987

988
	file = ERR_PTR(-ENOMEM);
989 990 991
	if (hugetlb_reserve_pages(inode, 0,
			size >> huge_page_shift(hstate_inode(inode)), NULL,
			acctflag))
992 993
		goto out_inode;

994
	d_instantiate(path.dentry, inode);
L
Linus Torvalds 已提交
995
	inode->i_size = size;
996
	clear_nlink(inode);
997

998
	file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
999
			&hugetlbfs_file_operations);
1000
	if (IS_ERR(file))
1001
		goto out_dentry; /* inode is already attached */
1002

L
Linus Torvalds 已提交
1003 1004
	return file;

1005 1006
out_inode:
	iput(inode);
L
Linus Torvalds 已提交
1007
out_dentry:
1008
	path_put(&path);
L
Linus Torvalds 已提交
1009
out_shm_unlock:
1010 1011 1012 1013
	if (*user) {
		user_shm_unlock(size, *user);
		*user = NULL;
	}
1014
	return file;
L
Linus Torvalds 已提交
1015 1016 1017 1018
}

static int __init init_hugetlbfs_fs(void)
{
1019
	struct hstate *h;
L
Linus Torvalds 已提交
1020
	int error;
1021
	int i;
L
Linus Torvalds 已提交
1022

1023
	if (!hugepages_supported()) {
1024
		pr_info("disabling because there are no supported hugepage sizes\n");
1025 1026 1027
		return -ENOTSUPP;
	}

1028
	error = -ENOMEM;
L
Linus Torvalds 已提交
1029 1030
	hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
					sizeof(struct hugetlbfs_inode_info),
1031
					0, 0, init_once);
L
Linus Torvalds 已提交
1032
	if (hugetlbfs_inode_cachep == NULL)
P
Peter Zijlstra 已提交
1033
		goto out2;
L
Linus Torvalds 已提交
1034 1035 1036 1037 1038

	error = register_filesystem(&hugetlbfs_fs_type);
	if (error)
		goto out;

1039 1040 1041 1042
	i = 0;
	for_each_hstate(h) {
		char buf[50];
		unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
L
Linus Torvalds 已提交
1043

1044 1045 1046
		snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
		hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
							buf);
L
Linus Torvalds 已提交
1047

1048
		if (IS_ERR(hugetlbfs_vfsmount[i])) {
1049
			pr_err("Cannot mount internal hugetlbfs for "
1050 1051 1052 1053 1054 1055 1056 1057 1058
				"page size %uK", ps_kb);
			error = PTR_ERR(hugetlbfs_vfsmount[i]);
			hugetlbfs_vfsmount[i] = NULL;
		}
		i++;
	}
	/* Non default hstates are optional */
	if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
		return 0;
L
Linus Torvalds 已提交
1059 1060

 out:
1061
	kmem_cache_destroy(hugetlbfs_inode_cachep);
P
Peter Zijlstra 已提交
1062
 out2:
L
Linus Torvalds 已提交
1063 1064 1065 1066 1067
	return error;
}

static void __exit exit_hugetlbfs_fs(void)
{
1068 1069 1070 1071
	struct hstate *h;
	int i;


1072 1073 1074 1075 1076
	/*
	 * Make sure all delayed rcu free inodes are flushed before we
	 * destroy cache.
	 */
	rcu_barrier();
L
Linus Torvalds 已提交
1077
	kmem_cache_destroy(hugetlbfs_inode_cachep);
1078 1079 1080
	i = 0;
	for_each_hstate(h)
		kern_unmount(hugetlbfs_vfsmount[i++]);
L
Linus Torvalds 已提交
1081 1082 1083 1084 1085 1086 1087
	unregister_filesystem(&hugetlbfs_fs_type);
}

module_init(init_hugetlbfs_fs)
module_exit(exit_hugetlbfs_fs)

MODULE_LICENSE("GPL");