task_mmu.c 38.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2
#include <linux/mm.h>
#include <linux/hugetlb.h>
3
#include <linux/huge_mm.h>
L
Linus Torvalds 已提交
4 5
#include <linux/mount.h>
#include <linux/seq_file.h>
M
Mauricio Lin 已提交
6
#include <linux/highmem.h>
K
Kees Cook 已提交
7
#include <linux/ptrace.h>
8
#include <linux/slab.h>
9 10
#include <linux/pagemap.h>
#include <linux/mempolicy.h>
11
#include <linux/rmap.h>
12 13
#include <linux/swap.h>
#include <linux/swapops.h>
14
#include <linux/mmu_notifier.h>
M
Mauricio Lin 已提交
15

L
Linus Torvalds 已提交
16 17
#include <asm/elf.h>
#include <asm/uaccess.h>
M
Mauricio Lin 已提交
18
#include <asm/tlbflush.h>
L
Linus Torvalds 已提交
19 20
#include "internal.h"

21
void task_mem(struct seq_file *m, struct mm_struct *mm)
L
Linus Torvalds 已提交
22
{
K
KAMEZAWA Hiroyuki 已提交
23
	unsigned long data, text, lib, swap;
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;

	/*
	 * Note: to minimize their overhead, mm maintains hiwater_vm and
	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
	 * collector of these hiwater stats must therefore get total_vm
	 * and rss too, which will usually be the higher.  Barriers? not
	 * worth the effort, such snapshots can always be inconsistent.
	 */
	hiwater_vm = total_vm = mm->total_vm;
	if (hiwater_vm < mm->hiwater_vm)
		hiwater_vm = mm->hiwater_vm;
	hiwater_rss = total_rss = get_mm_rss(mm);
	if (hiwater_rss < mm->hiwater_rss)
		hiwater_rss = mm->hiwater_rss;
L
Linus Torvalds 已提交
39 40 41 42

	data = mm->total_vm - mm->shared_vm - mm->stack_vm;
	text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
	lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
K
KAMEZAWA Hiroyuki 已提交
43
	swap = get_mm_counter(mm, MM_SWAPENTS);
44
	seq_printf(m,
45
		"VmPeak:\t%8lu kB\n"
L
Linus Torvalds 已提交
46 47
		"VmSize:\t%8lu kB\n"
		"VmLck:\t%8lu kB\n"
48
		"VmPin:\t%8lu kB\n"
49
		"VmHWM:\t%8lu kB\n"
L
Linus Torvalds 已提交
50 51 52 53 54
		"VmRSS:\t%8lu kB\n"
		"VmData:\t%8lu kB\n"
		"VmStk:\t%8lu kB\n"
		"VmExe:\t%8lu kB\n"
		"VmLib:\t%8lu kB\n"
K
KAMEZAWA Hiroyuki 已提交
55 56
		"VmPTE:\t%8lu kB\n"
		"VmSwap:\t%8lu kB\n",
57
		hiwater_vm << (PAGE_SHIFT-10),
58
		total_vm << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
59
		mm->locked_vm << (PAGE_SHIFT-10),
60
		mm->pinned_vm << (PAGE_SHIFT-10),
61 62
		hiwater_rss << (PAGE_SHIFT-10),
		total_rss << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
63 64
		data << (PAGE_SHIFT-10),
		mm->stack_vm << (PAGE_SHIFT-10), text, lib,
65 66
		(PTRS_PER_PTE * sizeof(pte_t) *
		 atomic_long_read(&mm->nr_ptes)) >> 10,
K
KAMEZAWA Hiroyuki 已提交
67
		swap << (PAGE_SHIFT-10));
L
Linus Torvalds 已提交
68 69 70 71 72 73 74
}

unsigned long task_vsize(struct mm_struct *mm)
{
	return PAGE_SIZE * mm->total_vm;
}

75 76 77
unsigned long task_statm(struct mm_struct *mm,
			 unsigned long *shared, unsigned long *text,
			 unsigned long *data, unsigned long *resident)
L
Linus Torvalds 已提交
78
{
K
KAMEZAWA Hiroyuki 已提交
79
	*shared = get_mm_counter(mm, MM_FILEPAGES);
L
Linus Torvalds 已提交
80 81 82
	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
								>> PAGE_SHIFT;
	*data = mm->total_vm - mm->shared_vm;
K
KAMEZAWA Hiroyuki 已提交
83
	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
L
Linus Torvalds 已提交
84 85 86 87 88 89 90 91 92 93 94
	return mm->total_vm;
}

static void pad_len_spaces(struct seq_file *m, int len)
{
	len = 25 + sizeof(void*) * 6 - len;
	if (len < 1)
		len = 1;
	seq_printf(m, "%*c", len, ' ');
}

95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
#ifdef CONFIG_NUMA
/*
 * These functions are for numa_maps but called in generic **maps seq_file
 * ->start(), ->stop() ops.
 *
 * numa_maps scans all vmas under mmap_sem and checks their mempolicy.
 * Each mempolicy object is controlled by reference counting. The problem here
 * is how to avoid accessing dead mempolicy object.
 *
 * Because we're holding mmap_sem while reading seq_file, it's safe to access
 * each vma's mempolicy, no vma objects will never drop refs to mempolicy.
 *
 * A task's mempolicy (task->mempolicy) has different behavior. task->mempolicy
 * is set and replaced under mmap_sem but unrefed and cleared under task_lock().
 * So, without task_lock(), we cannot trust get_vma_policy() because we cannot
 * gurantee the task never exits under us. But taking task_lock() around
 * get_vma_plicy() causes lock order problem.
 *
 * To access task->mempolicy without lock, we hold a reference count of an
 * object pointed by task->mempolicy and remember it. This will guarantee
 * that task->mempolicy points to an alive object or NULL in numa_maps accesses.
 */
static void hold_task_mempolicy(struct proc_maps_private *priv)
{
	struct task_struct *task = priv->task;

	task_lock(task);
	priv->task_mempolicy = task->mempolicy;
	mpol_get(priv->task_mempolicy);
	task_unlock(task);
}
static void release_task_mempolicy(struct proc_maps_private *priv)
{
	mpol_put(priv->task_mempolicy);
}
#else
static void hold_task_mempolicy(struct proc_maps_private *priv)
{
}
static void release_task_mempolicy(struct proc_maps_private *priv)
{
}
#endif

139 140 141 142
static void vma_stop(struct proc_maps_private *priv, struct vm_area_struct *vma)
{
	if (vma && vma != priv->tail_vma) {
		struct mm_struct *mm = vma->vm_mm;
143
		release_task_mempolicy(priv);
144 145 146 147
		up_read(&mm->mmap_sem);
		mmput(mm);
	}
}
148

149
static void *m_start(struct seq_file *m, loff_t *pos)
M
Mauricio Lin 已提交
150
{
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
	struct proc_maps_private *priv = m->private;
	unsigned long last_addr = m->version;
	struct mm_struct *mm;
	struct vm_area_struct *vma, *tail_vma = NULL;
	loff_t l = *pos;

	/* Clear the per syscall fields in priv */
	priv->task = NULL;
	priv->tail_vma = NULL;

	/*
	 * We remember last_addr rather than next_addr to hit with
	 * mmap_cache most of the time. We have zero last_addr at
	 * the beginning and also after lseek. We will have -1 last_addr
	 * after the end of the vmas.
	 */

	if (last_addr == -1UL)
		return NULL;

	priv->task = get_pid_task(priv->pid, PIDTYPE_PID);
	if (!priv->task)
A
Al Viro 已提交
173
		return ERR_PTR(-ESRCH);
174

C
Cong Wang 已提交
175
	mm = mm_access(priv->task, PTRACE_MODE_READ);
A
Al Viro 已提交
176 177
	if (!mm || IS_ERR(mm))
		return mm;
178
	down_read(&mm->mmap_sem);
179

180
	tail_vma = get_gate_vma(priv->task->mm);
181
	priv->tail_vma = tail_vma;
182
	hold_task_mempolicy(priv);
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
	/* Start with last addr hint */
	vma = find_vma(mm, last_addr);
	if (last_addr && vma) {
		vma = vma->vm_next;
		goto out;
	}

	/*
	 * Check the vma index is within the range and do
	 * sequential scan until m_index.
	 */
	vma = NULL;
	if ((unsigned long)l < mm->map_count) {
		vma = mm->mmap;
		while (l-- && vma)
			vma = vma->vm_next;
		goto out;
	}

	if (l != mm->map_count)
		tail_vma = NULL; /* After gate vma */

out:
	if (vma)
		return vma;

209
	release_task_mempolicy(priv);
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
	/* End of vmas has been reached */
	m->version = (tail_vma != NULL)? 0: -1UL;
	up_read(&mm->mmap_sem);
	mmput(mm);
	return tail_vma;
}

static void *m_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct proc_maps_private *priv = m->private;
	struct vm_area_struct *vma = v;
	struct vm_area_struct *tail_vma = priv->tail_vma;

	(*pos)++;
	if (vma && (vma != tail_vma) && vma->vm_next)
		return vma->vm_next;
	vma_stop(priv, vma);
	return (vma != tail_vma)? tail_vma: NULL;
}

static void m_stop(struct seq_file *m, void *v)
{
	struct proc_maps_private *priv = m->private;
	struct vm_area_struct *vma = v;

235 236
	if (!IS_ERR(vma))
		vma_stop(priv, vma);
237 238 239 240 241
	if (priv->task)
		put_task_struct(priv->task);
}

static int do_maps_open(struct inode *inode, struct file *file,
242
			const struct seq_operations *ops)
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
{
	struct proc_maps_private *priv;
	int ret = -ENOMEM;
	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	if (priv) {
		priv->pid = proc_pid(inode);
		ret = seq_open(file, ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = priv;
		} else {
			kfree(priv);
		}
	}
	return ret;
}
M
Mauricio Lin 已提交
259

260 261
static void
show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid)
L
Linus Torvalds 已提交
262
{
M
Mauricio Lin 已提交
263 264
	struct mm_struct *mm = vma->vm_mm;
	struct file *file = vma->vm_file;
265 266
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;
267
	vm_flags_t flags = vma->vm_flags;
L
Linus Torvalds 已提交
268
	unsigned long ino = 0;
269
	unsigned long long pgoff = 0;
270
	unsigned long start, end;
L
Linus Torvalds 已提交
271 272
	dev_t dev = 0;
	int len;
273
	const char *name = NULL;
L
Linus Torvalds 已提交
274 275

	if (file) {
A
Al Viro 已提交
276
		struct inode *inode = file_inode(vma->vm_file);
L
Linus Torvalds 已提交
277 278
		dev = inode->i_sb->s_dev;
		ino = inode->i_ino;
279
		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
L
Linus Torvalds 已提交
280 281
	}

282 283
	/* We don't show the stack guard page in /proc/maps */
	start = vma->vm_start;
284 285 286 287 288
	if (stack_guard_page_start(vma, start))
		start += PAGE_SIZE;
	end = vma->vm_end;
	if (stack_guard_page_end(vma, end))
		end -= PAGE_SIZE;
289

290
	seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n",
291
			start,
292
			end,
L
Linus Torvalds 已提交
293 294 295 296
			flags & VM_READ ? 'r' : '-',
			flags & VM_WRITE ? 'w' : '-',
			flags & VM_EXEC ? 'x' : '-',
			flags & VM_MAYSHARE ? 's' : 'p',
297
			pgoff,
L
Linus Torvalds 已提交
298 299 300 301 302 303
			MAJOR(dev), MINOR(dev), ino, &len);

	/*
	 * Print the dentry name for named mappings, and a
	 * special [heap] marker for the heap:
	 */
M
Mauricio Lin 已提交
304
	if (file) {
L
Linus Torvalds 已提交
305
		pad_len_spaces(m, len);
306
		seq_path(m, &file->f_path, "\n");
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
		goto done;
	}

	name = arch_vma_name(vma);
	if (!name) {
		pid_t tid;

		if (!mm) {
			name = "[vdso]";
			goto done;
		}

		if (vma->vm_start <= mm->brk &&
		    vma->vm_end >= mm->start_brk) {
			name = "[heap]";
			goto done;
		}

		tid = vm_is_stack(task, vma, is_pid);

		if (tid != 0) {
			/*
			 * Thread stack in /proc/PID/task/TID/maps or
			 * the main process stack.
			 */
			if (!is_pid || (vma->vm_start <= mm->start_stack &&
			    vma->vm_end >= mm->start_stack)) {
				name = "[stack]";
335
			} else {
336 337 338
				/* Thread stack in /proc/PID/maps */
				pad_len_spaces(m, len);
				seq_printf(m, "[stack:%d]", tid);
L
Linus Torvalds 已提交
339
			}
340
		}
341 342 343 344 345 346
	}

done:
	if (name) {
		pad_len_spaces(m, len);
		seq_puts(m, name);
L
Linus Torvalds 已提交
347 348
	}
	seq_putc(m, '\n');
349 350
}

351
static int show_map(struct seq_file *m, void *v, int is_pid)
352 353 354 355 356
{
	struct vm_area_struct *vma = v;
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;

357
	show_map_vma(m, vma, is_pid);
M
Mauricio Lin 已提交
358 359

	if (m->count < m->size)  /* vma is copied successfully */
360 361
		m->version = (vma != get_gate_vma(task->mm))
			? vma->vm_start : 0;
L
Linus Torvalds 已提交
362 363 364
	return 0;
}

365 366 367 368 369 370 371 372 373 374
static int show_pid_map(struct seq_file *m, void *v)
{
	return show_map(m, v, 1);
}

static int show_tid_map(struct seq_file *m, void *v)
{
	return show_map(m, v, 0);
}

375
static const struct seq_operations proc_pid_maps_op = {
376 377 378
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
379 380 381 382 383 384 385 386
	.show	= show_pid_map
};

static const struct seq_operations proc_tid_maps_op = {
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= show_tid_map
387 388
};

389
static int pid_maps_open(struct inode *inode, struct file *file)
390 391 392 393
{
	return do_maps_open(inode, file, &proc_pid_maps_op);
}

394 395 396 397 398 399 400 401 402 403 404 405 406 407
static int tid_maps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_tid_maps_op);
}

const struct file_operations proc_pid_maps_operations = {
	.open		= pid_maps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

const struct file_operations proc_tid_maps_operations = {
	.open		= tid_maps_open,
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

/*
 * Proportional Set Size(PSS): my share of RSS.
 *
 * PSS of a process is the count of pages it has in memory, where each
 * page is divided by the number of processes sharing it.  So if a
 * process has 1000 pages all to itself, and 1000 shared with one other
 * process, its PSS will be 1500.
 *
 * To keep (accumulated) division errors low, we adopt a 64bit
 * fixed-point pss counter to minimize division errors. So (pss >>
 * PSS_SHIFT) would be the real byte count.
 *
 * A shift of 12 before division means (assuming 4K page size):
 * 	- 1M 3-user-pages add up to 8KB errors;
 * 	- supports mapcount up to 2^24, or 16M;
 * 	- supports PSS up to 2^52 bytes, or 4PB.
 */
#define PSS_SHIFT 12

432
#ifdef CONFIG_PROC_PAGE_MONITOR
P
Peter Zijlstra 已提交
433
struct mem_size_stats {
434 435 436 437 438 439 440
	struct vm_area_struct *vma;
	unsigned long resident;
	unsigned long shared_clean;
	unsigned long shared_dirty;
	unsigned long private_clean;
	unsigned long private_dirty;
	unsigned long referenced;
441
	unsigned long anonymous;
442
	unsigned long anonymous_thp;
P
Peter Zijlstra 已提交
443
	unsigned long swap;
444
	unsigned long nonlinear;
445 446 447
	u64 pss;
};

448 449

static void smaps_pte_entry(pte_t ptent, unsigned long addr,
450
		unsigned long ptent_size, struct mm_walk *walk)
451 452 453
{
	struct mem_size_stats *mss = walk->private;
	struct vm_area_struct *vma = mss->vma;
454
	pgoff_t pgoff = linear_page_index(vma, addr);
455
	struct page *page = NULL;
456 457
	int mapcount;

458 459 460 461
	if (pte_present(ptent)) {
		page = vm_normal_page(vma, addr, ptent);
	} else if (is_swap_pte(ptent)) {
		swp_entry_t swpent = pte_to_swp_entry(ptent);
462

463 464 465 466
		if (!non_swap_entry(swpent))
			mss->swap += ptent_size;
		else if (is_migration_entry(swpent))
			page = migration_entry_to_page(swpent);
467 468 469
	} else if (pte_file(ptent)) {
		if (pte_to_pgoff(ptent) != pgoff)
			mss->nonlinear += ptent_size;
470
	}
471 472 473 474 475

	if (!page)
		return;

	if (PageAnon(page))
476
		mss->anonymous += ptent_size;
477

478 479 480
	if (page->index != pgoff)
		mss->nonlinear += ptent_size;

481
	mss->resident += ptent_size;
482 483
	/* Accumulate the size in pages that have been accessed. */
	if (pte_young(ptent) || PageReferenced(page))
484
		mss->referenced += ptent_size;
485 486 487
	mapcount = page_mapcount(page);
	if (mapcount >= 2) {
		if (pte_dirty(ptent) || PageDirty(page))
488
			mss->shared_dirty += ptent_size;
489
		else
490 491
			mss->shared_clean += ptent_size;
		mss->pss += (ptent_size << PSS_SHIFT) / mapcount;
492 493
	} else {
		if (pte_dirty(ptent) || PageDirty(page))
494
			mss->private_dirty += ptent_size;
495
		else
496 497
			mss->private_clean += ptent_size;
		mss->pss += (ptent_size << PSS_SHIFT);
498 499 500
	}
}

501
static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
D
Dave Hansen 已提交
502
			   struct mm_walk *walk)
M
Mauricio Lin 已提交
503
{
D
Dave Hansen 已提交
504
	struct mem_size_stats *mss = walk->private;
505
	struct vm_area_struct *vma = mss->vma;
506
	pte_t *pte;
507
	spinlock_t *ptl;
M
Mauricio Lin 已提交
508

509 510
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
		smaps_pte_entry(*(pte_t *)pmd, addr, HPAGE_PMD_SIZE, walk);
511
		spin_unlock(&walk->mm->page_table_lock);
512 513
		mss->anonymous_thp += HPAGE_PMD_SIZE;
		return 0;
514
	}
515 516 517

	if (pmd_trans_unstable(pmd))
		return 0;
518 519 520 521 522
	/*
	 * The mmap_sem held all the way back in m_start() is what
	 * keeps khugepaged out of here and from collapsing things
	 * in here.
	 */
523
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
524
	for (; addr != end; pte++, addr += PAGE_SIZE)
525
		smaps_pte_entry(*pte, addr, PAGE_SIZE, walk);
526 527
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
528
	return 0;
M
Mauricio Lin 已提交
529 530
}

531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
{
	/*
	 * Don't forget to update Documentation/ on changes.
	 */
	static const char mnemonics[BITS_PER_LONG][2] = {
		/*
		 * In case if we meet a flag we don't know about.
		 */
		[0 ... (BITS_PER_LONG-1)] = "??",

		[ilog2(VM_READ)]	= "rd",
		[ilog2(VM_WRITE)]	= "wr",
		[ilog2(VM_EXEC)]	= "ex",
		[ilog2(VM_SHARED)]	= "sh",
		[ilog2(VM_MAYREAD)]	= "mr",
		[ilog2(VM_MAYWRITE)]	= "mw",
		[ilog2(VM_MAYEXEC)]	= "me",
		[ilog2(VM_MAYSHARE)]	= "ms",
		[ilog2(VM_GROWSDOWN)]	= "gd",
		[ilog2(VM_PFNMAP)]	= "pf",
		[ilog2(VM_DENYWRITE)]	= "dw",
		[ilog2(VM_LOCKED)]	= "lo",
		[ilog2(VM_IO)]		= "io",
		[ilog2(VM_SEQ_READ)]	= "sr",
		[ilog2(VM_RAND_READ)]	= "rr",
		[ilog2(VM_DONTCOPY)]	= "dc",
		[ilog2(VM_DONTEXPAND)]	= "de",
		[ilog2(VM_ACCOUNT)]	= "ac",
		[ilog2(VM_NORESERVE)]	= "nr",
		[ilog2(VM_HUGETLB)]	= "ht",
		[ilog2(VM_NONLINEAR)]	= "nl",
		[ilog2(VM_ARCH_1)]	= "ar",
		[ilog2(VM_DONTDUMP)]	= "dd",
565 566 567
#ifdef CONFIG_MEM_SOFT_DIRTY
		[ilog2(VM_SOFTDIRTY)]	= "sd",
#endif
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
		[ilog2(VM_MIXEDMAP)]	= "mm",
		[ilog2(VM_HUGEPAGE)]	= "hg",
		[ilog2(VM_NOHUGEPAGE)]	= "nh",
		[ilog2(VM_MERGEABLE)]	= "mg",
	};
	size_t i;

	seq_puts(m, "VmFlags: ");
	for (i = 0; i < BITS_PER_LONG; i++) {
		if (vma->vm_flags & (1UL << i)) {
			seq_printf(m, "%c%c ",
				   mnemonics[i][0], mnemonics[i][1]);
		}
	}
	seq_putc(m, '\n');
}

585
static int show_smap(struct seq_file *m, void *v, int is_pid)
M
Mauricio Lin 已提交
586
{
587 588
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;
M
Mauricio Lin 已提交
589 590
	struct vm_area_struct *vma = v;
	struct mem_size_stats mss;
D
Dave Hansen 已提交
591 592 593 594 595
	struct mm_walk smaps_walk = {
		.pmd_entry = smaps_pte_range,
		.mm = vma->vm_mm,
		.private = &mss,
	};
M
Mauricio Lin 已提交
596 597

	memset(&mss, 0, sizeof mss);
598
	mss.vma = vma;
599
	/* mmap_sem is held in m_start */
N
Nick Piggin 已提交
600
	if (vma->vm_mm && !is_vm_hugetlb_page(vma))
D
Dave Hansen 已提交
601
		walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
602

603
	show_map_vma(m, vma, is_pid);
604 605 606 607 608 609 610 611 612

	seq_printf(m,
		   "Size:           %8lu kB\n"
		   "Rss:            %8lu kB\n"
		   "Pss:            %8lu kB\n"
		   "Shared_Clean:   %8lu kB\n"
		   "Shared_Dirty:   %8lu kB\n"
		   "Private_Clean:  %8lu kB\n"
		   "Private_Dirty:  %8lu kB\n"
P
Peter Zijlstra 已提交
613
		   "Referenced:     %8lu kB\n"
614
		   "Anonymous:      %8lu kB\n"
615
		   "AnonHugePages:  %8lu kB\n"
616
		   "Swap:           %8lu kB\n"
617
		   "KernelPageSize: %8lu kB\n"
618 619
		   "MMUPageSize:    %8lu kB\n"
		   "Locked:         %8lu kB\n",
620 621 622 623 624 625 626
		   (vma->vm_end - vma->vm_start) >> 10,
		   mss.resident >> 10,
		   (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
		   mss.shared_clean  >> 10,
		   mss.shared_dirty  >> 10,
		   mss.private_clean >> 10,
		   mss.private_dirty >> 10,
P
Peter Zijlstra 已提交
627
		   mss.referenced >> 10,
628
		   mss.anonymous >> 10,
629
		   mss.anonymous_thp >> 10,
630
		   mss.swap >> 10,
631
		   vma_kernel_pagesize(vma) >> 10,
632 633 634
		   vma_mmu_pagesize(vma) >> 10,
		   (vma->vm_flags & VM_LOCKED) ?
			(unsigned long)(mss.pss >> (10 + PSS_SHIFT)) : 0);
635

636 637 638 639
	if (vma->vm_flags & VM_NONLINEAR)
		seq_printf(m, "Nonlinear:      %8lu kB\n",
				mss.nonlinear >> 10);

640 641
	show_smap_vma_flags(m, vma);

642
	if (m->count < m->size)  /* vma is copied successfully */
643 644
		m->version = (vma != get_gate_vma(task->mm))
			? vma->vm_start : 0;
645
	return 0;
M
Mauricio Lin 已提交
646 647
}

648 649 650 651 652 653 654 655 656 657
static int show_pid_smap(struct seq_file *m, void *v)
{
	return show_smap(m, v, 1);
}

static int show_tid_smap(struct seq_file *m, void *v)
{
	return show_smap(m, v, 0);
}

658
static const struct seq_operations proc_pid_smaps_op = {
659 660 661
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
662 663 664 665 666 667 668 669
	.show	= show_pid_smap
};

static const struct seq_operations proc_tid_smaps_op = {
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= show_tid_smap
670 671
};

672
static int pid_smaps_open(struct inode *inode, struct file *file)
673 674 675 676
{
	return do_maps_open(inode, file, &proc_pid_smaps_op);
}

677 678 679 680 681 682 683 684 685 686 687 688 689 690
static int tid_smaps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_tid_smaps_op);
}

const struct file_operations proc_pid_smaps_operations = {
	.open		= pid_smaps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

const struct file_operations proc_tid_smaps_operations = {
	.open		= tid_smaps_open,
691 692 693 694 695
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
/*
 * We do not want to have constant page-shift bits sitting in
 * pagemap entries and are about to reuse them some time soon.
 *
 * Here's the "migration strategy":
 * 1. when the system boots these bits remain what they are,
 *    but a warning about future change is printed in log;
 * 2. once anyone clears soft-dirty bits via clear_refs file,
 *    these flag is set to denote, that user is aware of the
 *    new API and those page-shift bits change their meaning.
 *    The respective warning is printed in dmesg;
 * 3. In a couple of releases we will remove all the mentions
 *    of page-shift in pagemap entries.
 */

static bool soft_dirty_cleared __read_mostly;

713 714 715 716
enum clear_refs_types {
	CLEAR_REFS_ALL = 1,
	CLEAR_REFS_ANON,
	CLEAR_REFS_MAPPED,
717
	CLEAR_REFS_SOFT_DIRTY,
718 719 720
	CLEAR_REFS_LAST,
};

721 722
struct clear_refs_private {
	struct vm_area_struct *vma;
723
	enum clear_refs_types type;
724 725
};

726 727 728 729 730 731 732 733 734 735 736
static inline void clear_soft_dirty(struct vm_area_struct *vma,
		unsigned long addr, pte_t *pte)
{
#ifdef CONFIG_MEM_SOFT_DIRTY
	/*
	 * The soft-dirty tracker uses #PF-s to catch writes
	 * to pages, so write-protect the pte as well. See the
	 * Documentation/vm/soft-dirty.txt for full description
	 * of how soft-dirty works.
	 */
	pte_t ptent = *pte;
737 738 739 740 741 742

	if (pte_present(ptent)) {
		ptent = pte_wrprotect(ptent);
		ptent = pte_clear_flags(ptent, _PAGE_SOFT_DIRTY);
	} else if (is_swap_pte(ptent)) {
		ptent = pte_swp_clear_soft_dirty(ptent);
743 744
	} else if (pte_file(ptent)) {
		ptent = pte_file_clear_soft_dirty(ptent);
745 746
	}

747 748 749
	if (vma->vm_flags & VM_SOFTDIRTY)
		vma->vm_flags &= ~VM_SOFTDIRTY;

750 751 752 753
	set_pte_at(vma->vm_mm, addr, pte, ptent);
#endif
}

754
static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
D
Dave Hansen 已提交
755
				unsigned long end, struct mm_walk *walk)
756
{
757 758
	struct clear_refs_private *cp = walk->private;
	struct vm_area_struct *vma = cp->vma;
759 760 761 762
	pte_t *pte, ptent;
	spinlock_t *ptl;
	struct page *page;

763
	split_huge_page_pmd(vma, addr, pmd);
764 765
	if (pmd_trans_unstable(pmd))
		return 0;
766

767 768 769 770
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE) {
		ptent = *pte;

771 772 773 774 775
		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
			clear_soft_dirty(vma, addr, pte);
			continue;
		}

776 777 778
		if (!pte_present(ptent))
			continue;

779 780 781 782 783 784 785 786 787 788 789 790 791
		page = vm_normal_page(vma, addr, ptent);
		if (!page)
			continue;

		/* Clear accessed and referenced bits. */
		ptep_test_and_clear_young(vma, addr, pte);
		ClearPageReferenced(page);
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
	return 0;
}

792 793
static ssize_t clear_refs_write(struct file *file, const char __user *buf,
				size_t count, loff_t *ppos)
794
{
795
	struct task_struct *task;
796
	char buffer[PROC_NUMBUF];
797
	struct mm_struct *mm;
798
	struct vm_area_struct *vma;
799 800
	enum clear_refs_types type;
	int itype;
A
Alexey Dobriyan 已提交
801
	int rv;
802

803 804 805 806 807
	memset(buffer, 0, sizeof(buffer));
	if (count > sizeof(buffer) - 1)
		count = sizeof(buffer) - 1;
	if (copy_from_user(buffer, buf, count))
		return -EFAULT;
808
	rv = kstrtoint(strstrip(buffer), 10, &itype);
A
Alexey Dobriyan 已提交
809 810
	if (rv < 0)
		return rv;
811 812
	type = (enum clear_refs_types)itype;
	if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
813
		return -EINVAL;
814 815 816 817 818 819 820

	if (type == CLEAR_REFS_SOFT_DIRTY) {
		soft_dirty_cleared = true;
		pr_warn_once("The pagemap bits 55-60 has changed their meaning! "
				"See the linux/Documentation/vm/pagemap.txt for details.\n");
	}

A
Al Viro 已提交
821
	task = get_proc_task(file_inode(file));
822 823 824 825
	if (!task)
		return -ESRCH;
	mm = get_task_mm(task);
	if (mm) {
826
		struct clear_refs_private cp = {
827
			.type = type,
828
		};
829 830 831
		struct mm_walk clear_refs_walk = {
			.pmd_entry = clear_refs_pte_range,
			.mm = mm,
832
			.private = &cp,
833
		};
834
		down_read(&mm->mmap_sem);
835 836
		if (type == CLEAR_REFS_SOFT_DIRTY)
			mmu_notifier_invalidate_range_start(mm, 0, -1);
D
Dave Hansen 已提交
837
		for (vma = mm->mmap; vma; vma = vma->vm_next) {
838
			cp.vma = vma;
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
			if (is_vm_hugetlb_page(vma))
				continue;
			/*
			 * Writing 1 to /proc/pid/clear_refs affects all pages.
			 *
			 * Writing 2 to /proc/pid/clear_refs only affects
			 * Anonymous pages.
			 *
			 * Writing 3 to /proc/pid/clear_refs only affects file
			 * mapped pages.
			 */
			if (type == CLEAR_REFS_ANON && vma->vm_file)
				continue;
			if (type == CLEAR_REFS_MAPPED && !vma->vm_file)
				continue;
			walk_page_range(vma->vm_start, vma->vm_end,
					&clear_refs_walk);
D
Dave Hansen 已提交
856
		}
857 858
		if (type == CLEAR_REFS_SOFT_DIRTY)
			mmu_notifier_invalidate_range_end(mm, 0, -1);
859 860 861 862 863
		flush_tlb_mm(mm);
		up_read(&mm->mmap_sem);
		mmput(mm);
	}
	put_task_struct(task);
864 865

	return count;
866 867
}

868 869
const struct file_operations proc_clear_refs_operations = {
	.write		= clear_refs_write,
870
	.llseek		= noop_llseek,
871 872
};

873 874 875 876
typedef struct {
	u64 pme;
} pagemap_entry_t;

877
struct pagemapread {
878
	int pos, len;		/* units: PM_ENTRY_BYTES, not bytes */
879
	pagemap_entry_t *buffer;
880
	bool v2;
881 882
};

883 884 885
#define PAGEMAP_WALK_SIZE	(PMD_SIZE)
#define PAGEMAP_WALK_MASK	(PMD_MASK)

886
#define PM_ENTRY_BYTES      sizeof(pagemap_entry_t)
887 888 889 890 891 892 893
#define PM_STATUS_BITS      3
#define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)
#define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
#define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
#define PM_PSHIFT_BITS      6
#define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
#define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
894
#define __PM_PSHIFT(x)      (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
895 896
#define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
#define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)
897 898
/* in "new" pagemap pshift bits are occupied with more status bits */
#define PM_STATUS2(v2, x)   (__PM_PSHIFT(v2 ? x : PAGE_SHIFT))
899

900
#define __PM_SOFT_DIRTY      (1LL)
901 902
#define PM_PRESENT          PM_STATUS(4LL)
#define PM_SWAP             PM_STATUS(2LL)
903
#define PM_FILE             PM_STATUS(1LL)
904
#define PM_NOT_PRESENT(v2)  PM_STATUS2(v2, 0)
905 906
#define PM_END_OF_BUFFER    1

907 908 909 910 911 912
static inline pagemap_entry_t make_pme(u64 val)
{
	return (pagemap_entry_t) { .pme = val };
}

static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
913 914
			  struct pagemapread *pm)
{
915
	pm->buffer[pm->pos++] = *pme;
916
	if (pm->pos >= pm->len)
917
		return PM_END_OF_BUFFER;
918 919 920 921
	return 0;
}

static int pagemap_pte_hole(unsigned long start, unsigned long end,
D
Dave Hansen 已提交
922
				struct mm_walk *walk)
923
{
D
Dave Hansen 已提交
924
	struct pagemapread *pm = walk->private;
925 926
	unsigned long addr;
	int err = 0;
927
	pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));
928

929
	for (addr = start; addr < end; addr += PAGE_SIZE) {
930
		err = add_to_pagemap(addr, &pme, pm);
931 932 933 934 935 936
		if (err)
			break;
	}
	return err;
}

937
static void pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
938
		struct vm_area_struct *vma, unsigned long addr, pte_t pte)
939
{
940 941
	u64 frame, flags;
	struct page *page = NULL;
942
	int flags2 = 0;
943

944 945 946 947
	if (pte_present(pte)) {
		frame = pte_pfn(pte);
		flags = PM_PRESENT;
		page = vm_normal_page(vma, addr, pte);
948 949
		if (pte_soft_dirty(pte))
			flags2 |= __PM_SOFT_DIRTY;
950
	} else if (is_swap_pte(pte)) {
951 952 953 954
		swp_entry_t entry;
		if (pte_swp_soft_dirty(pte))
			flags2 |= __PM_SOFT_DIRTY;
		entry = pte_to_swp_entry(pte);
955 956 957 958 959 960
		frame = swp_type(entry) |
			(swp_offset(entry) << MAX_SWAPFILES_SHIFT);
		flags = PM_SWAP;
		if (is_migration_entry(entry))
			page = migration_entry_to_page(entry);
	} else {
961 962 963
		if (vma->vm_flags & VM_SOFTDIRTY)
			flags2 |= __PM_SOFT_DIRTY;
		*pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, flags2));
964 965 966 967 968
		return;
	}

	if (page && !PageAnon(page))
		flags |= PM_FILE;
969
	if ((vma->vm_flags & VM_SOFTDIRTY))
970
		flags2 |= __PM_SOFT_DIRTY;
971

972
	*pme = make_pme(PM_PFRAME(frame) | PM_STATUS2(pm->v2, flags2) | flags);
973 974
}

975
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
976
static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
977
		pmd_t pmd, int offset, int pmd_flags2)
978 979 980 981 982 983 984
{
	/*
	 * Currently pmd for thp is always present because thp can not be
	 * swapped-out, migrated, or HWPOISONed (split in such cases instead.)
	 * This if-check is just to prepare for future implementation.
	 */
	if (pmd_present(pmd))
985
		*pme = make_pme(PM_PFRAME(pmd_pfn(pmd) + offset)
986
				| PM_STATUS2(pm->v2, pmd_flags2) | PM_PRESENT);
987
	else
988
		*pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, pmd_flags2));
989 990
}
#else
991
static inline void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
992
		pmd_t pmd, int offset, int pmd_flags2)
993 994 995 996
{
}
#endif

997
static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
D
Dave Hansen 已提交
998
			     struct mm_walk *walk)
999
{
1000
	struct vm_area_struct *vma;
D
Dave Hansen 已提交
1001
	struct pagemapread *pm = walk->private;
1002 1003
	pte_t *pte;
	int err = 0;
1004
	pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));
1005

1006 1007
	/* find the first VMA at or above 'addr' */
	vma = find_vma(walk->mm, addr);
1008
	if (vma && pmd_trans_huge_lock(pmd, vma) == 1) {
1009 1010
		int pmd_flags2;

1011 1012 1013 1014 1015
		if ((vma->vm_flags & VM_SOFTDIRTY) || pmd_soft_dirty(*pmd))
			pmd_flags2 = __PM_SOFT_DIRTY;
		else
			pmd_flags2 = 0;

1016 1017 1018 1019 1020
		for (; addr != end; addr += PAGE_SIZE) {
			unsigned long offset;

			offset = (addr & ~PAGEMAP_WALK_MASK) >>
					PAGE_SHIFT;
1021
			thp_pmd_to_pagemap_entry(&pme, pm, *pmd, offset, pmd_flags2);
1022
			err = add_to_pagemap(addr, &pme, pm);
1023 1024
			if (err)
				break;
1025 1026
		}
		spin_unlock(&walk->mm->page_table_lock);
1027
		return err;
1028 1029
	}

1030 1031
	if (pmd_trans_unstable(pmd))
		return 0;
1032
	for (; addr != end; addr += PAGE_SIZE) {
1033
		int flags2;
1034 1035 1036

		/* check to see if we've left 'vma' behind
		 * and need a new, higher one */
1037
		if (vma && (addr >= vma->vm_end)) {
1038
			vma = find_vma(walk->mm, addr);
1039 1040 1041 1042 1043
			if (vma && (vma->vm_flags & VM_SOFTDIRTY))
				flags2 = __PM_SOFT_DIRTY;
			else
				flags2 = 0;
			pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, flags2));
1044
		}
1045 1046 1047 1048 1049 1050

		/* check that 'vma' actually covers this address,
		 * and that it isn't a huge page vma */
		if (vma && (vma->vm_start <= addr) &&
		    !is_vm_hugetlb_page(vma)) {
			pte = pte_offset_map(pmd, addr);
1051
			pte_to_pagemap_entry(&pme, pm, vma, addr, *pte);
1052 1053 1054
			/* unmap before userspace copy */
			pte_unmap(pte);
		}
1055
		err = add_to_pagemap(addr, &pme, pm);
1056 1057 1058 1059 1060 1061 1062 1063 1064
		if (err)
			return err;
	}

	cond_resched();

	return err;
}

1065
#ifdef CONFIG_HUGETLB_PAGE
1066
static void huge_pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
1067
					pte_t pte, int offset, int flags2)
1068 1069
{
	if (pte_present(pte))
1070 1071 1072
		*pme = make_pme(PM_PFRAME(pte_pfn(pte) + offset)	|
				PM_STATUS2(pm->v2, flags2)		|
				PM_PRESENT);
1073
	else
1074 1075
		*pme = make_pme(PM_NOT_PRESENT(pm->v2)			|
				PM_STATUS2(pm->v2, flags2));
1076 1077
}

1078 1079 1080 1081
/* This function walks within one hugetlb entry in the single call */
static int pagemap_hugetlb_range(pte_t *pte, unsigned long hmask,
				 unsigned long addr, unsigned long end,
				 struct mm_walk *walk)
1082 1083
{
	struct pagemapread *pm = walk->private;
1084
	struct vm_area_struct *vma;
1085
	int err = 0;
1086
	int flags2;
1087
	pagemap_entry_t pme;
1088

1089 1090 1091 1092 1093 1094 1095 1096
	vma = find_vma(walk->mm, addr);
	WARN_ON_ONCE(!vma);

	if (vma && (vma->vm_flags & VM_SOFTDIRTY))
		flags2 = __PM_SOFT_DIRTY;
	else
		flags2 = 0;

1097
	for (; addr != end; addr += PAGE_SIZE) {
1098
		int offset = (addr & ~hmask) >> PAGE_SHIFT;
1099
		huge_pte_to_pagemap_entry(&pme, pm, *pte, offset, flags2);
1100
		err = add_to_pagemap(addr, &pme, pm);
1101 1102 1103 1104 1105 1106 1107 1108
		if (err)
			return err;
	}

	cond_resched();

	return err;
}
1109
#endif /* HUGETLB_PAGE */
1110

1111 1112 1113
/*
 * /proc/pid/pagemap - an array mapping virtual pages to pfns
 *
1114 1115 1116
 * For each page in the address space, this file contains one 64-bit entry
 * consisting of the following:
 *
1117
 * Bits 0-54  page frame number (PFN) if present
1118
 * Bits 0-4   swap type if swapped
1119
 * Bits 5-54  swap offset if swapped
1120
 * Bits 55-60 page shift (page size = 1<<page shift)
1121
 * Bit  61    page is file-page or shared-anon
1122 1123 1124 1125 1126 1127
 * Bit  62    page swapped
 * Bit  63    page present
 *
 * If the page is not present but in swap, then the PFN contains an
 * encoding of the swap file number and the page's offset into the
 * swap. Unmapped pages return a null PFN. This allows determining
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
 * precisely which pages are mapped (or in swap) and comparing mapped
 * pages between processes.
 *
 * Efficient users of this interface will use /proc/pid/maps to
 * determine which areas of memory are actually mapped and llseek to
 * skip over unmapped regions.
 */
static ssize_t pagemap_read(struct file *file, char __user *buf,
			    size_t count, loff_t *ppos)
{
A
Al Viro 已提交
1138
	struct task_struct *task = get_proc_task(file_inode(file));
1139 1140 1141
	struct mm_struct *mm;
	struct pagemapread pm;
	int ret = -ESRCH;
1142
	struct mm_walk pagemap_walk = {};
1143 1144 1145 1146
	unsigned long src;
	unsigned long svpfn;
	unsigned long start_vaddr;
	unsigned long end_vaddr;
1147
	int copied = 0;
1148 1149 1150 1151 1152 1153

	if (!task)
		goto out;

	ret = -EINVAL;
	/* file position must be aligned */
1154
	if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1155
		goto out_task;
1156 1157

	ret = 0;
1158 1159 1160
	if (!count)
		goto out_task;

1161
	pm.v2 = soft_dirty_cleared;
1162 1163
	pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
	pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);
1164
	ret = -ENOMEM;
1165
	if (!pm.buffer)
1166 1167
		goto out_task;

C
Cong Wang 已提交
1168
	mm = mm_access(task, PTRACE_MODE_READ);
1169 1170 1171
	ret = PTR_ERR(mm);
	if (!mm || IS_ERR(mm))
		goto out_free;
1172

1173 1174
	pagemap_walk.pmd_entry = pagemap_pte_range;
	pagemap_walk.pte_hole = pagemap_pte_hole;
1175
#ifdef CONFIG_HUGETLB_PAGE
1176
	pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1177
#endif
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
	pagemap_walk.mm = mm;
	pagemap_walk.private = &pm;

	src = *ppos;
	svpfn = src / PM_ENTRY_BYTES;
	start_vaddr = svpfn << PAGE_SHIFT;
	end_vaddr = TASK_SIZE_OF(task);

	/* watch out for wraparound */
	if (svpfn > TASK_SIZE_OF(task) >> PAGE_SHIFT)
		start_vaddr = end_vaddr;

	/*
	 * The odds are that this will stop walking way
	 * before end_vaddr, because the length of the
	 * user buffer is tracked in "pm", and the walk
	 * will stop when we hit the end of the buffer.
	 */
1196 1197 1198 1199 1200 1201
	ret = 0;
	while (count && (start_vaddr < end_vaddr)) {
		int len;
		unsigned long end;

		pm.pos = 0;
1202
		end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1203 1204 1205 1206 1207 1208 1209 1210 1211
		/* overflow ? */
		if (end < start_vaddr || end > end_vaddr)
			end = end_vaddr;
		down_read(&mm->mmap_sem);
		ret = walk_page_range(start_vaddr, end, &pagemap_walk);
		up_read(&mm->mmap_sem);
		start_vaddr = end;

		len = min(count, PM_ENTRY_BYTES * pm.pos);
1212
		if (copy_to_user(buf, pm.buffer, len)) {
1213
			ret = -EFAULT;
1214
			goto out_mm;
1215 1216 1217 1218
		}
		copied += len;
		buf += len;
		count -= len;
1219
	}
1220 1221 1222 1223
	*ppos += copied;
	if (!ret || ret == PM_END_OF_BUFFER)
		ret = copied;

1224 1225
out_mm:
	mmput(mm);
1226 1227
out_free:
	kfree(pm.buffer);
1228 1229 1230 1231 1232 1233
out_task:
	put_task_struct(task);
out:
	return ret;
}

1234 1235 1236 1237 1238 1239 1240 1241
static int pagemap_open(struct inode *inode, struct file *file)
{
	pr_warn_once("Bits 55-60 of /proc/PID/pagemap entries are about "
			"to stop being page-shift some time soon. See the "
			"linux/Documentation/vm/pagemap.txt for details.\n");
	return 0;
}

1242 1243 1244
const struct file_operations proc_pagemap_operations = {
	.llseek		= mem_lseek, /* borrow this */
	.read		= pagemap_read,
1245
	.open		= pagemap_open,
1246
};
1247
#endif /* CONFIG_PROC_PAGE_MONITOR */
1248

1249 1250
#ifdef CONFIG_NUMA

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
struct numa_maps {
	struct vm_area_struct *vma;
	unsigned long pages;
	unsigned long anon;
	unsigned long active;
	unsigned long writeback;
	unsigned long mapcount_max;
	unsigned long dirty;
	unsigned long swapcache;
	unsigned long node[MAX_NUMNODES];
};

1263 1264 1265 1266 1267
struct numa_maps_private {
	struct proc_maps_private proc_maps;
	struct numa_maps md;
};

1268 1269
static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
			unsigned long nr_pages)
1270 1271 1272
{
	int count = page_mapcount(page);

1273
	md->pages += nr_pages;
1274
	if (pte_dirty || PageDirty(page))
1275
		md->dirty += nr_pages;
1276 1277

	if (PageSwapCache(page))
1278
		md->swapcache += nr_pages;
1279 1280

	if (PageActive(page) || PageUnevictable(page))
1281
		md->active += nr_pages;
1282 1283

	if (PageWriteback(page))
1284
		md->writeback += nr_pages;
1285 1286

	if (PageAnon(page))
1287
		md->anon += nr_pages;
1288 1289 1290 1291

	if (count > md->mapcount_max)
		md->mapcount_max = count;

1292
	md->node[page_to_nid(page)] += nr_pages;
1293 1294
}

1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
		unsigned long addr)
{
	struct page *page;
	int nid;

	if (!pte_present(pte))
		return NULL;

	page = vm_normal_page(vma, addr, pte);
	if (!page)
		return NULL;

	if (PageReserved(page))
		return NULL;

	nid = page_to_nid(page);
1312
	if (!node_isset(nid, node_states[N_MEMORY]))
1313 1314 1315 1316 1317
		return NULL;

	return page;
}

1318 1319 1320 1321 1322 1323 1324 1325 1326
static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
		unsigned long end, struct mm_walk *walk)
{
	struct numa_maps *md;
	spinlock_t *ptl;
	pte_t *orig_pte;
	pte_t *pte;

	md = walk->private;
1327 1328 1329 1330 1331 1332 1333 1334 1335

	if (pmd_trans_huge_lock(pmd, md->vma) == 1) {
		pte_t huge_pte = *(pte_t *)pmd;
		struct page *page;

		page = can_gather_numa_stats(huge_pte, md->vma, addr);
		if (page)
			gather_stats(page, md, pte_dirty(huge_pte),
				     HPAGE_PMD_SIZE/PAGE_SIZE);
1336
		spin_unlock(&walk->mm->page_table_lock);
1337
		return 0;
1338 1339
	}

1340 1341
	if (pmd_trans_unstable(pmd))
		return 0;
1342 1343
	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
	do {
1344
		struct page *page = can_gather_numa_stats(*pte, md->vma, addr);
1345 1346
		if (!page)
			continue;
1347
		gather_stats(page, md, pte_dirty(*pte), 1);
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367

	} while (pte++, addr += PAGE_SIZE, addr != end);
	pte_unmap_unlock(orig_pte, ptl);
	return 0;
}
#ifdef CONFIG_HUGETLB_PAGE
static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
		unsigned long addr, unsigned long end, struct mm_walk *walk)
{
	struct numa_maps *md;
	struct page *page;

	if (pte_none(*pte))
		return 0;

	page = pte_page(*pte);
	if (!page)
		return 0;

	md = walk->private;
1368
	gather_stats(page, md, pte_dirty(*pte), 1);
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
	return 0;
}

#else
static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
		unsigned long addr, unsigned long end, struct mm_walk *walk)
{
	return 0;
}
#endif

/*
 * Display pages allocated per node and memory policy via /proc.
 */
1383
static int show_numa_map(struct seq_file *m, void *v, int is_pid)
1384
{
1385 1386
	struct numa_maps_private *numa_priv = m->private;
	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1387
	struct vm_area_struct *vma = v;
1388
	struct numa_maps *md = &numa_priv->md;
1389
	struct file *file = vma->vm_file;
1390
	struct task_struct *task = proc_priv->task;
1391 1392 1393
	struct mm_struct *mm = vma->vm_mm;
	struct mm_walk walk = {};
	struct mempolicy *pol;
1394 1395
	char buffer[64];
	int nid;
1396 1397 1398 1399

	if (!mm)
		return 0;

1400 1401
	/* Ensure we start with an empty set of numa_maps statistics. */
	memset(md, 0, sizeof(*md));
1402 1403 1404 1405 1406 1407 1408 1409

	md->vma = vma;

	walk.hugetlb_entry = gather_hugetbl_stats;
	walk.pmd_entry = gather_pte_stats;
	walk.private = md;
	walk.mm = mm;

1410
	pol = get_vma_policy(task, vma, vma->vm_start);
1411
	mpol_to_str(buffer, sizeof(buffer), pol);
1412 1413 1414 1415 1416 1417 1418 1419 1420
	mpol_cond_put(pol);

	seq_printf(m, "%08lx %s", vma->vm_start, buffer);

	if (file) {
		seq_printf(m, " file=");
		seq_path(m, &file->f_path, "\n\t= ");
	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
		seq_printf(m, " heap");
1421
	} else {
1422
		pid_t tid = vm_is_stack(task, vma, is_pid);
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
		if (tid != 0) {
			/*
			 * Thread stack in /proc/PID/task/TID/maps or
			 * the main process stack.
			 */
			if (!is_pid || (vma->vm_start <= mm->start_stack &&
			    vma->vm_end >= mm->start_stack))
				seq_printf(m, " stack");
			else
				seq_printf(m, " stack:%d", tid);
		}
1434 1435
	}

1436 1437 1438
	if (is_vm_hugetlb_page(vma))
		seq_printf(m, " huge");

1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
	walk_page_range(vma->vm_start, vma->vm_end, &walk);

	if (!md->pages)
		goto out;

	if (md->anon)
		seq_printf(m, " anon=%lu", md->anon);

	if (md->dirty)
		seq_printf(m, " dirty=%lu", md->dirty);

	if (md->pages != md->anon && md->pages != md->dirty)
		seq_printf(m, " mapped=%lu", md->pages);

	if (md->mapcount_max > 1)
		seq_printf(m, " mapmax=%lu", md->mapcount_max);

	if (md->swapcache)
		seq_printf(m, " swapcache=%lu", md->swapcache);

	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
		seq_printf(m, " active=%lu", md->active);

	if (md->writeback)
		seq_printf(m, " writeback=%lu", md->writeback);

1465 1466 1467
	for_each_node_state(nid, N_MEMORY)
		if (md->node[nid])
			seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1468 1469 1470 1471
out:
	seq_putc(m, '\n');

	if (m->count < m->size)
1472
		m->version = (vma != proc_priv->tail_vma) ? vma->vm_start : 0;
1473 1474
	return 0;
}
1475

1476 1477 1478 1479 1480 1481 1482 1483 1484 1485
static int show_pid_numa_map(struct seq_file *m, void *v)
{
	return show_numa_map(m, v, 1);
}

static int show_tid_numa_map(struct seq_file *m, void *v)
{
	return show_numa_map(m, v, 0);
}

1486
static const struct seq_operations proc_pid_numa_maps_op = {
1487 1488 1489 1490
	.start  = m_start,
	.next   = m_next,
	.stop   = m_stop,
	.show   = show_pid_numa_map,
1491
};
1492

1493 1494 1495 1496 1497 1498 1499 1500 1501
static const struct seq_operations proc_tid_numa_maps_op = {
	.start  = m_start,
	.next   = m_next,
	.stop   = m_stop,
	.show   = show_tid_numa_map,
};

static int numa_maps_open(struct inode *inode, struct file *file,
			  const struct seq_operations *ops)
1502
{
1503 1504 1505 1506 1507
	struct numa_maps_private *priv;
	int ret = -ENOMEM;
	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	if (priv) {
		priv->proc_maps.pid = proc_pid(inode);
1508
		ret = seq_open(file, ops);
1509 1510 1511 1512 1513 1514 1515 1516
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = priv;
		} else {
			kfree(priv);
		}
	}
	return ret;
1517 1518
}

1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
static int pid_numa_maps_open(struct inode *inode, struct file *file)
{
	return numa_maps_open(inode, file, &proc_pid_numa_maps_op);
}

static int tid_numa_maps_open(struct inode *inode, struct file *file)
{
	return numa_maps_open(inode, file, &proc_tid_numa_maps_op);
}

const struct file_operations proc_pid_numa_maps_operations = {
	.open		= pid_numa_maps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

const struct file_operations proc_tid_numa_maps_operations = {
	.open		= tid_numa_maps_open,
1538 1539
	.read		= seq_read,
	.llseek		= seq_lseek,
1540
	.release	= seq_release_private,
1541
};
1542
#endif /* CONFIG_NUMA */