task_mmu.c 33.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2
#include <linux/mm.h>
#include <linux/hugetlb.h>
3
#include <linux/huge_mm.h>
L
Linus Torvalds 已提交
4 5
#include <linux/mount.h>
#include <linux/seq_file.h>
M
Mauricio Lin 已提交
6
#include <linux/highmem.h>
K
Kees Cook 已提交
7
#include <linux/ptrace.h>
8
#include <linux/slab.h>
9 10
#include <linux/pagemap.h>
#include <linux/mempolicy.h>
11
#include <linux/rmap.h>
12 13
#include <linux/swap.h>
#include <linux/swapops.h>
M
Mauricio Lin 已提交
14

L
Linus Torvalds 已提交
15 16
#include <asm/elf.h>
#include <asm/uaccess.h>
M
Mauricio Lin 已提交
17
#include <asm/tlbflush.h>
L
Linus Torvalds 已提交
18 19
#include "internal.h"

20
void task_mem(struct seq_file *m, struct mm_struct *mm)
L
Linus Torvalds 已提交
21
{
K
KAMEZAWA Hiroyuki 已提交
22
	unsigned long data, text, lib, swap;
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;

	/*
	 * Note: to minimize their overhead, mm maintains hiwater_vm and
	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
	 * collector of these hiwater stats must therefore get total_vm
	 * and rss too, which will usually be the higher.  Barriers? not
	 * worth the effort, such snapshots can always be inconsistent.
	 */
	hiwater_vm = total_vm = mm->total_vm;
	if (hiwater_vm < mm->hiwater_vm)
		hiwater_vm = mm->hiwater_vm;
	hiwater_rss = total_rss = get_mm_rss(mm);
	if (hiwater_rss < mm->hiwater_rss)
		hiwater_rss = mm->hiwater_rss;
L
Linus Torvalds 已提交
38 39 40 41

	data = mm->total_vm - mm->shared_vm - mm->stack_vm;
	text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
	lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
K
KAMEZAWA Hiroyuki 已提交
42
	swap = get_mm_counter(mm, MM_SWAPENTS);
43
	seq_printf(m,
44
		"VmPeak:\t%8lu kB\n"
L
Linus Torvalds 已提交
45 46
		"VmSize:\t%8lu kB\n"
		"VmLck:\t%8lu kB\n"
47
		"VmPin:\t%8lu kB\n"
48
		"VmHWM:\t%8lu kB\n"
L
Linus Torvalds 已提交
49 50 51 52 53
		"VmRSS:\t%8lu kB\n"
		"VmData:\t%8lu kB\n"
		"VmStk:\t%8lu kB\n"
		"VmExe:\t%8lu kB\n"
		"VmLib:\t%8lu kB\n"
K
KAMEZAWA Hiroyuki 已提交
54 55
		"VmPTE:\t%8lu kB\n"
		"VmSwap:\t%8lu kB\n",
56
		hiwater_vm << (PAGE_SHIFT-10),
57
		total_vm << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
58
		mm->locked_vm << (PAGE_SHIFT-10),
59
		mm->pinned_vm << (PAGE_SHIFT-10),
60 61
		hiwater_rss << (PAGE_SHIFT-10),
		total_rss << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
62 63
		data << (PAGE_SHIFT-10),
		mm->stack_vm << (PAGE_SHIFT-10), text, lib,
K
KAMEZAWA Hiroyuki 已提交
64 65
		(PTRS_PER_PTE*sizeof(pte_t)*mm->nr_ptes) >> 10,
		swap << (PAGE_SHIFT-10));
L
Linus Torvalds 已提交
66 67 68 69 70 71 72
}

unsigned long task_vsize(struct mm_struct *mm)
{
	return PAGE_SIZE * mm->total_vm;
}

73 74 75
unsigned long task_statm(struct mm_struct *mm,
			 unsigned long *shared, unsigned long *text,
			 unsigned long *data, unsigned long *resident)
L
Linus Torvalds 已提交
76
{
K
KAMEZAWA Hiroyuki 已提交
77
	*shared = get_mm_counter(mm, MM_FILEPAGES);
L
Linus Torvalds 已提交
78 79 80
	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
								>> PAGE_SHIFT;
	*data = mm->total_vm - mm->shared_vm;
K
KAMEZAWA Hiroyuki 已提交
81
	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
L
Linus Torvalds 已提交
82 83 84 85 86 87 88 89 90 91 92
	return mm->total_vm;
}

static void pad_len_spaces(struct seq_file *m, int len)
{
	len = 25 + sizeof(void*) * 6 - len;
	if (len < 1)
		len = 1;
	seq_printf(m, "%*c", len, ' ');
}

93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
#ifdef CONFIG_NUMA
/*
 * These functions are for numa_maps but called in generic **maps seq_file
 * ->start(), ->stop() ops.
 *
 * numa_maps scans all vmas under mmap_sem and checks their mempolicy.
 * Each mempolicy object is controlled by reference counting. The problem here
 * is how to avoid accessing dead mempolicy object.
 *
 * Because we're holding mmap_sem while reading seq_file, it's safe to access
 * each vma's mempolicy, no vma objects will never drop refs to mempolicy.
 *
 * A task's mempolicy (task->mempolicy) has different behavior. task->mempolicy
 * is set and replaced under mmap_sem but unrefed and cleared under task_lock().
 * So, without task_lock(), we cannot trust get_vma_policy() because we cannot
 * gurantee the task never exits under us. But taking task_lock() around
 * get_vma_plicy() causes lock order problem.
 *
 * To access task->mempolicy without lock, we hold a reference count of an
 * object pointed by task->mempolicy and remember it. This will guarantee
 * that task->mempolicy points to an alive object or NULL in numa_maps accesses.
 */
static void hold_task_mempolicy(struct proc_maps_private *priv)
{
	struct task_struct *task = priv->task;

	task_lock(task);
	priv->task_mempolicy = task->mempolicy;
	mpol_get(priv->task_mempolicy);
	task_unlock(task);
}
static void release_task_mempolicy(struct proc_maps_private *priv)
{
	mpol_put(priv->task_mempolicy);
}
#else
static void hold_task_mempolicy(struct proc_maps_private *priv)
{
}
static void release_task_mempolicy(struct proc_maps_private *priv)
{
}
#endif

137 138 139 140
static void vma_stop(struct proc_maps_private *priv, struct vm_area_struct *vma)
{
	if (vma && vma != priv->tail_vma) {
		struct mm_struct *mm = vma->vm_mm;
141
		release_task_mempolicy(priv);
142 143 144 145
		up_read(&mm->mmap_sem);
		mmput(mm);
	}
}
146

147
static void *m_start(struct seq_file *m, loff_t *pos)
M
Mauricio Lin 已提交
148
{
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
	struct proc_maps_private *priv = m->private;
	unsigned long last_addr = m->version;
	struct mm_struct *mm;
	struct vm_area_struct *vma, *tail_vma = NULL;
	loff_t l = *pos;

	/* Clear the per syscall fields in priv */
	priv->task = NULL;
	priv->tail_vma = NULL;

	/*
	 * We remember last_addr rather than next_addr to hit with
	 * mmap_cache most of the time. We have zero last_addr at
	 * the beginning and also after lseek. We will have -1 last_addr
	 * after the end of the vmas.
	 */

	if (last_addr == -1UL)
		return NULL;

	priv->task = get_pid_task(priv->pid, PIDTYPE_PID);
	if (!priv->task)
A
Al Viro 已提交
171
		return ERR_PTR(-ESRCH);
172

C
Cong Wang 已提交
173
	mm = mm_access(priv->task, PTRACE_MODE_READ);
A
Al Viro 已提交
174 175
	if (!mm || IS_ERR(mm))
		return mm;
176
	down_read(&mm->mmap_sem);
177

178
	tail_vma = get_gate_vma(priv->task->mm);
179
	priv->tail_vma = tail_vma;
180
	hold_task_mempolicy(priv);
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
	/* Start with last addr hint */
	vma = find_vma(mm, last_addr);
	if (last_addr && vma) {
		vma = vma->vm_next;
		goto out;
	}

	/*
	 * Check the vma index is within the range and do
	 * sequential scan until m_index.
	 */
	vma = NULL;
	if ((unsigned long)l < mm->map_count) {
		vma = mm->mmap;
		while (l-- && vma)
			vma = vma->vm_next;
		goto out;
	}

	if (l != mm->map_count)
		tail_vma = NULL; /* After gate vma */

out:
	if (vma)
		return vma;

207
	release_task_mempolicy(priv);
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
	/* End of vmas has been reached */
	m->version = (tail_vma != NULL)? 0: -1UL;
	up_read(&mm->mmap_sem);
	mmput(mm);
	return tail_vma;
}

static void *m_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct proc_maps_private *priv = m->private;
	struct vm_area_struct *vma = v;
	struct vm_area_struct *tail_vma = priv->tail_vma;

	(*pos)++;
	if (vma && (vma != tail_vma) && vma->vm_next)
		return vma->vm_next;
	vma_stop(priv, vma);
	return (vma != tail_vma)? tail_vma: NULL;
}

static void m_stop(struct seq_file *m, void *v)
{
	struct proc_maps_private *priv = m->private;
	struct vm_area_struct *vma = v;

233 234
	if (!IS_ERR(vma))
		vma_stop(priv, vma);
235 236 237 238 239
	if (priv->task)
		put_task_struct(priv->task);
}

static int do_maps_open(struct inode *inode, struct file *file,
240
			const struct seq_operations *ops)
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
{
	struct proc_maps_private *priv;
	int ret = -ENOMEM;
	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	if (priv) {
		priv->pid = proc_pid(inode);
		ret = seq_open(file, ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = priv;
		} else {
			kfree(priv);
		}
	}
	return ret;
}
M
Mauricio Lin 已提交
257

258 259
static void
show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid)
L
Linus Torvalds 已提交
260
{
M
Mauricio Lin 已提交
261 262
	struct mm_struct *mm = vma->vm_mm;
	struct file *file = vma->vm_file;
263 264
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;
265
	vm_flags_t flags = vma->vm_flags;
L
Linus Torvalds 已提交
266
	unsigned long ino = 0;
267
	unsigned long long pgoff = 0;
268
	unsigned long start, end;
L
Linus Torvalds 已提交
269 270
	dev_t dev = 0;
	int len;
271
	const char *name = NULL;
L
Linus Torvalds 已提交
272 273

	if (file) {
274
		struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
275 276
		dev = inode->i_sb->s_dev;
		ino = inode->i_ino;
277
		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
L
Linus Torvalds 已提交
278 279
	}

280 281
	/* We don't show the stack guard page in /proc/maps */
	start = vma->vm_start;
282 283 284 285 286
	if (stack_guard_page_start(vma, start))
		start += PAGE_SIZE;
	end = vma->vm_end;
	if (stack_guard_page_end(vma, end))
		end -= PAGE_SIZE;
287

288
	seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n",
289
			start,
290
			end,
L
Linus Torvalds 已提交
291 292 293 294
			flags & VM_READ ? 'r' : '-',
			flags & VM_WRITE ? 'w' : '-',
			flags & VM_EXEC ? 'x' : '-',
			flags & VM_MAYSHARE ? 's' : 'p',
295
			pgoff,
L
Linus Torvalds 已提交
296 297 298 299 300 301
			MAJOR(dev), MINOR(dev), ino, &len);

	/*
	 * Print the dentry name for named mappings, and a
	 * special [heap] marker for the heap:
	 */
M
Mauricio Lin 已提交
302
	if (file) {
L
Linus Torvalds 已提交
303
		pad_len_spaces(m, len);
304
		seq_path(m, &file->f_path, "\n");
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
		goto done;
	}

	name = arch_vma_name(vma);
	if (!name) {
		pid_t tid;

		if (!mm) {
			name = "[vdso]";
			goto done;
		}

		if (vma->vm_start <= mm->brk &&
		    vma->vm_end >= mm->start_brk) {
			name = "[heap]";
			goto done;
		}

		tid = vm_is_stack(task, vma, is_pid);

		if (tid != 0) {
			/*
			 * Thread stack in /proc/PID/task/TID/maps or
			 * the main process stack.
			 */
			if (!is_pid || (vma->vm_start <= mm->start_stack &&
			    vma->vm_end >= mm->start_stack)) {
				name = "[stack]";
333
			} else {
334 335 336
				/* Thread stack in /proc/PID/maps */
				pad_len_spaces(m, len);
				seq_printf(m, "[stack:%d]", tid);
L
Linus Torvalds 已提交
337
			}
338
		}
339 340 341 342 343 344
	}

done:
	if (name) {
		pad_len_spaces(m, len);
		seq_puts(m, name);
L
Linus Torvalds 已提交
345 346
	}
	seq_putc(m, '\n');
347 348
}

349
static int show_map(struct seq_file *m, void *v, int is_pid)
350 351 352 353 354
{
	struct vm_area_struct *vma = v;
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;

355
	show_map_vma(m, vma, is_pid);
M
Mauricio Lin 已提交
356 357

	if (m->count < m->size)  /* vma is copied successfully */
358 359
		m->version = (vma != get_gate_vma(task->mm))
			? vma->vm_start : 0;
L
Linus Torvalds 已提交
360 361 362
	return 0;
}

363 364 365 366 367 368 369 370 371 372
static int show_pid_map(struct seq_file *m, void *v)
{
	return show_map(m, v, 1);
}

static int show_tid_map(struct seq_file *m, void *v)
{
	return show_map(m, v, 0);
}

373
static const struct seq_operations proc_pid_maps_op = {
374 375 376
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
377 378 379 380 381 382 383 384
	.show	= show_pid_map
};

static const struct seq_operations proc_tid_maps_op = {
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= show_tid_map
385 386
};

387
static int pid_maps_open(struct inode *inode, struct file *file)
388 389 390 391
{
	return do_maps_open(inode, file, &proc_pid_maps_op);
}

392 393 394 395 396 397 398 399 400 401 402 403 404 405
static int tid_maps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_tid_maps_op);
}

const struct file_operations proc_pid_maps_operations = {
	.open		= pid_maps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

const struct file_operations proc_tid_maps_operations = {
	.open		= tid_maps_open,
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

/*
 * Proportional Set Size(PSS): my share of RSS.
 *
 * PSS of a process is the count of pages it has in memory, where each
 * page is divided by the number of processes sharing it.  So if a
 * process has 1000 pages all to itself, and 1000 shared with one other
 * process, its PSS will be 1500.
 *
 * To keep (accumulated) division errors low, we adopt a 64bit
 * fixed-point pss counter to minimize division errors. So (pss >>
 * PSS_SHIFT) would be the real byte count.
 *
 * A shift of 12 before division means (assuming 4K page size):
 * 	- 1M 3-user-pages add up to 8KB errors;
 * 	- supports mapcount up to 2^24, or 16M;
 * 	- supports PSS up to 2^52 bytes, or 4PB.
 */
#define PSS_SHIFT 12

430
#ifdef CONFIG_PROC_PAGE_MONITOR
P
Peter Zijlstra 已提交
431
struct mem_size_stats {
432 433 434 435 436 437 438
	struct vm_area_struct *vma;
	unsigned long resident;
	unsigned long shared_clean;
	unsigned long shared_dirty;
	unsigned long private_clean;
	unsigned long private_dirty;
	unsigned long referenced;
439
	unsigned long anonymous;
440
	unsigned long anonymous_thp;
P
Peter Zijlstra 已提交
441
	unsigned long swap;
442
	unsigned long nonlinear;
443 444 445
	u64 pss;
};

446 447

static void smaps_pte_entry(pte_t ptent, unsigned long addr,
448
		unsigned long ptent_size, struct mm_walk *walk)
449 450 451
{
	struct mem_size_stats *mss = walk->private;
	struct vm_area_struct *vma = mss->vma;
452
	pgoff_t pgoff = linear_page_index(vma, addr);
453
	struct page *page = NULL;
454 455
	int mapcount;

456 457 458 459
	if (pte_present(ptent)) {
		page = vm_normal_page(vma, addr, ptent);
	} else if (is_swap_pte(ptent)) {
		swp_entry_t swpent = pte_to_swp_entry(ptent);
460

461 462 463 464
		if (!non_swap_entry(swpent))
			mss->swap += ptent_size;
		else if (is_migration_entry(swpent))
			page = migration_entry_to_page(swpent);
465 466 467
	} else if (pte_file(ptent)) {
		if (pte_to_pgoff(ptent) != pgoff)
			mss->nonlinear += ptent_size;
468
	}
469 470 471 472 473

	if (!page)
		return;

	if (PageAnon(page))
474
		mss->anonymous += ptent_size;
475

476 477 478
	if (page->index != pgoff)
		mss->nonlinear += ptent_size;

479
	mss->resident += ptent_size;
480 481
	/* Accumulate the size in pages that have been accessed. */
	if (pte_young(ptent) || PageReferenced(page))
482
		mss->referenced += ptent_size;
483 484 485
	mapcount = page_mapcount(page);
	if (mapcount >= 2) {
		if (pte_dirty(ptent) || PageDirty(page))
486
			mss->shared_dirty += ptent_size;
487
		else
488 489
			mss->shared_clean += ptent_size;
		mss->pss += (ptent_size << PSS_SHIFT) / mapcount;
490 491
	} else {
		if (pte_dirty(ptent) || PageDirty(page))
492
			mss->private_dirty += ptent_size;
493
		else
494 495
			mss->private_clean += ptent_size;
		mss->pss += (ptent_size << PSS_SHIFT);
496 497 498
	}
}

499
static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
D
Dave Hansen 已提交
500
			   struct mm_walk *walk)
M
Mauricio Lin 已提交
501
{
D
Dave Hansen 已提交
502
	struct mem_size_stats *mss = walk->private;
503
	struct vm_area_struct *vma = mss->vma;
504
	pte_t *pte;
505
	spinlock_t *ptl;
M
Mauricio Lin 已提交
506

507 508
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
		smaps_pte_entry(*(pte_t *)pmd, addr, HPAGE_PMD_SIZE, walk);
509
		spin_unlock(&walk->mm->page_table_lock);
510 511
		mss->anonymous_thp += HPAGE_PMD_SIZE;
		return 0;
512
	}
513 514 515

	if (pmd_trans_unstable(pmd))
		return 0;
516 517 518 519 520
	/*
	 * The mmap_sem held all the way back in m_start() is what
	 * keeps khugepaged out of here and from collapsing things
	 * in here.
	 */
521
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
522
	for (; addr != end; pte++, addr += PAGE_SIZE)
523
		smaps_pte_entry(*pte, addr, PAGE_SIZE, walk);
524 525
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
526
	return 0;
M
Mauricio Lin 已提交
527 528
}

529
static int show_smap(struct seq_file *m, void *v, int is_pid)
M
Mauricio Lin 已提交
530
{
531 532
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;
M
Mauricio Lin 已提交
533 534
	struct vm_area_struct *vma = v;
	struct mem_size_stats mss;
D
Dave Hansen 已提交
535 536 537 538 539
	struct mm_walk smaps_walk = {
		.pmd_entry = smaps_pte_range,
		.mm = vma->vm_mm,
		.private = &mss,
	};
M
Mauricio Lin 已提交
540 541

	memset(&mss, 0, sizeof mss);
542
	mss.vma = vma;
543
	/* mmap_sem is held in m_start */
N
Nick Piggin 已提交
544
	if (vma->vm_mm && !is_vm_hugetlb_page(vma))
D
Dave Hansen 已提交
545
		walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
546

547
	show_map_vma(m, vma, is_pid);
548 549 550 551 552 553 554 555 556

	seq_printf(m,
		   "Size:           %8lu kB\n"
		   "Rss:            %8lu kB\n"
		   "Pss:            %8lu kB\n"
		   "Shared_Clean:   %8lu kB\n"
		   "Shared_Dirty:   %8lu kB\n"
		   "Private_Clean:  %8lu kB\n"
		   "Private_Dirty:  %8lu kB\n"
P
Peter Zijlstra 已提交
557
		   "Referenced:     %8lu kB\n"
558
		   "Anonymous:      %8lu kB\n"
559
		   "AnonHugePages:  %8lu kB\n"
560
		   "Swap:           %8lu kB\n"
561
		   "KernelPageSize: %8lu kB\n"
562 563
		   "MMUPageSize:    %8lu kB\n"
		   "Locked:         %8lu kB\n",
564 565 566 567 568 569 570
		   (vma->vm_end - vma->vm_start) >> 10,
		   mss.resident >> 10,
		   (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
		   mss.shared_clean  >> 10,
		   mss.shared_dirty  >> 10,
		   mss.private_clean >> 10,
		   mss.private_dirty >> 10,
P
Peter Zijlstra 已提交
571
		   mss.referenced >> 10,
572
		   mss.anonymous >> 10,
573
		   mss.anonymous_thp >> 10,
574
		   mss.swap >> 10,
575
		   vma_kernel_pagesize(vma) >> 10,
576 577 578
		   vma_mmu_pagesize(vma) >> 10,
		   (vma->vm_flags & VM_LOCKED) ?
			(unsigned long)(mss.pss >> (10 + PSS_SHIFT)) : 0);
579

580 581 582 583
	if (vma->vm_flags & VM_NONLINEAR)
		seq_printf(m, "Nonlinear:      %8lu kB\n",
				mss.nonlinear >> 10);

584
	if (m->count < m->size)  /* vma is copied successfully */
585 586
		m->version = (vma != get_gate_vma(task->mm))
			? vma->vm_start : 0;
587
	return 0;
M
Mauricio Lin 已提交
588 589
}

590 591 592 593 594 595 596 597 598 599
static int show_pid_smap(struct seq_file *m, void *v)
{
	return show_smap(m, v, 1);
}

static int show_tid_smap(struct seq_file *m, void *v)
{
	return show_smap(m, v, 0);
}

600
static const struct seq_operations proc_pid_smaps_op = {
601 602 603
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
604 605 606 607 608 609 610 611
	.show	= show_pid_smap
};

static const struct seq_operations proc_tid_smaps_op = {
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= show_tid_smap
612 613
};

614
static int pid_smaps_open(struct inode *inode, struct file *file)
615 616 617 618
{
	return do_maps_open(inode, file, &proc_pid_smaps_op);
}

619 620 621 622 623 624 625 626 627 628 629 630 631 632
static int tid_smaps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_tid_smaps_op);
}

const struct file_operations proc_pid_smaps_operations = {
	.open		= pid_smaps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

const struct file_operations proc_tid_smaps_operations = {
	.open		= tid_smaps_open,
633 634 635 636 637 638
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
D
Dave Hansen 已提交
639
				unsigned long end, struct mm_walk *walk)
640
{
D
Dave Hansen 已提交
641
	struct vm_area_struct *vma = walk->private;
642 643 644 645
	pte_t *pte, ptent;
	spinlock_t *ptl;
	struct page *page;

646
	split_huge_page_pmd(walk->mm, pmd);
647 648
	if (pmd_trans_unstable(pmd))
		return 0;
649

650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE) {
		ptent = *pte;
		if (!pte_present(ptent))
			continue;

		page = vm_normal_page(vma, addr, ptent);
		if (!page)
			continue;

		/* Clear accessed and referenced bits. */
		ptep_test_and_clear_young(vma, addr, pte);
		ClearPageReferenced(page);
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
	return 0;
}

669 670 671 672
#define CLEAR_REFS_ALL 1
#define CLEAR_REFS_ANON 2
#define CLEAR_REFS_MAPPED 3

673 674
static ssize_t clear_refs_write(struct file *file, const char __user *buf,
				size_t count, loff_t *ppos)
675
{
676
	struct task_struct *task;
677
	char buffer[PROC_NUMBUF];
678
	struct mm_struct *mm;
679
	struct vm_area_struct *vma;
A
Alexey Dobriyan 已提交
680 681
	int type;
	int rv;
682

683 684 685 686 687
	memset(buffer, 0, sizeof(buffer));
	if (count > sizeof(buffer) - 1)
		count = sizeof(buffer) - 1;
	if (copy_from_user(buffer, buf, count))
		return -EFAULT;
A
Alexey Dobriyan 已提交
688 689 690
	rv = kstrtoint(strstrip(buffer), 10, &type);
	if (rv < 0)
		return rv;
691
	if (type < CLEAR_REFS_ALL || type > CLEAR_REFS_MAPPED)
692 693 694 695 696 697
		return -EINVAL;
	task = get_proc_task(file->f_path.dentry->d_inode);
	if (!task)
		return -ESRCH;
	mm = get_task_mm(task);
	if (mm) {
698 699 700 701
		struct mm_walk clear_refs_walk = {
			.pmd_entry = clear_refs_pte_range,
			.mm = mm,
		};
702
		down_read(&mm->mmap_sem);
D
Dave Hansen 已提交
703 704
		for (vma = mm->mmap; vma; vma = vma->vm_next) {
			clear_refs_walk.private = vma;
705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
			if (is_vm_hugetlb_page(vma))
				continue;
			/*
			 * Writing 1 to /proc/pid/clear_refs affects all pages.
			 *
			 * Writing 2 to /proc/pid/clear_refs only affects
			 * Anonymous pages.
			 *
			 * Writing 3 to /proc/pid/clear_refs only affects file
			 * mapped pages.
			 */
			if (type == CLEAR_REFS_ANON && vma->vm_file)
				continue;
			if (type == CLEAR_REFS_MAPPED && !vma->vm_file)
				continue;
			walk_page_range(vma->vm_start, vma->vm_end,
					&clear_refs_walk);
D
Dave Hansen 已提交
722
		}
723 724 725 726 727
		flush_tlb_mm(mm);
		up_read(&mm->mmap_sem);
		mmput(mm);
	}
	put_task_struct(task);
728 729

	return count;
730 731
}

732 733
const struct file_operations proc_clear_refs_operations = {
	.write		= clear_refs_write,
734
	.llseek		= noop_llseek,
735 736
};

737 738 739 740
typedef struct {
	u64 pme;
} pagemap_entry_t;

741
struct pagemapread {
742
	int pos, len;
743
	pagemap_entry_t *buffer;
744 745
};

746 747 748
#define PAGEMAP_WALK_SIZE	(PMD_SIZE)
#define PAGEMAP_WALK_MASK	(PMD_MASK)

749 750 751 752 753 754 755 756 757 758 759 760 761 762
#define PM_ENTRY_BYTES      sizeof(u64)
#define PM_STATUS_BITS      3
#define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)
#define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
#define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
#define PM_PSHIFT_BITS      6
#define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
#define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
#define PM_PSHIFT(x)        (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
#define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
#define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)

#define PM_PRESENT          PM_STATUS(4LL)
#define PM_SWAP             PM_STATUS(2LL)
763
#define PM_FILE             PM_STATUS(1LL)
764
#define PM_NOT_PRESENT      PM_PSHIFT(PAGE_SHIFT)
765 766
#define PM_END_OF_BUFFER    1

767 768 769 770 771 772
static inline pagemap_entry_t make_pme(u64 val)
{
	return (pagemap_entry_t) { .pme = val };
}

static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
773 774
			  struct pagemapread *pm)
{
775
	pm->buffer[pm->pos++] = *pme;
776
	if (pm->pos >= pm->len)
777
		return PM_END_OF_BUFFER;
778 779 780 781
	return 0;
}

static int pagemap_pte_hole(unsigned long start, unsigned long end,
D
Dave Hansen 已提交
782
				struct mm_walk *walk)
783
{
D
Dave Hansen 已提交
784
	struct pagemapread *pm = walk->private;
785 786
	unsigned long addr;
	int err = 0;
787 788
	pagemap_entry_t pme = make_pme(PM_NOT_PRESENT);

789
	for (addr = start; addr < end; addr += PAGE_SIZE) {
790
		err = add_to_pagemap(addr, &pme, pm);
791 792 793 794 795 796
		if (err)
			break;
	}
	return err;
}

797 798
static void pte_to_pagemap_entry(pagemap_entry_t *pme,
		struct vm_area_struct *vma, unsigned long addr, pte_t pte)
799
{
800 801
	u64 frame, flags;
	struct page *page = NULL;
802

803 804 805 806 807 808 809 810 811 812 813 814 815
	if (pte_present(pte)) {
		frame = pte_pfn(pte);
		flags = PM_PRESENT;
		page = vm_normal_page(vma, addr, pte);
	} else if (is_swap_pte(pte)) {
		swp_entry_t entry = pte_to_swp_entry(pte);

		frame = swp_type(entry) |
			(swp_offset(entry) << MAX_SWAPFILES_SHIFT);
		flags = PM_SWAP;
		if (is_migration_entry(entry))
			page = migration_entry_to_page(entry);
	} else {
816
		*pme = make_pme(PM_NOT_PRESENT);
817 818 819 820 821 822 823
		return;
	}

	if (page && !PageAnon(page))
		flags |= PM_FILE;

	*pme = make_pme(PM_PFRAME(frame) | PM_PSHIFT(PAGE_SHIFT) | flags);
824 825
}

826
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
827 828
static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme,
					pmd_t pmd, int offset)
829 830 831 832 833 834 835
{
	/*
	 * Currently pmd for thp is always present because thp can not be
	 * swapped-out, migrated, or HWPOISONed (split in such cases instead.)
	 * This if-check is just to prepare for future implementation.
	 */
	if (pmd_present(pmd))
836 837
		*pme = make_pme(PM_PFRAME(pmd_pfn(pmd) + offset)
				| PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT);
838 839
	else
		*pme = make_pme(PM_NOT_PRESENT);
840 841
}
#else
842 843
static inline void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme,
						pmd_t pmd, int offset)
844 845 846 847
{
}
#endif

848
static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
D
Dave Hansen 已提交
849
			     struct mm_walk *walk)
850
{
851
	struct vm_area_struct *vma;
D
Dave Hansen 已提交
852
	struct pagemapread *pm = walk->private;
853 854
	pte_t *pte;
	int err = 0;
855
	pagemap_entry_t pme = make_pme(PM_NOT_PRESENT);
856

857 858
	/* find the first VMA at or above 'addr' */
	vma = find_vma(walk->mm, addr);
859
	if (vma && pmd_trans_huge_lock(pmd, vma) == 1) {
860 861 862 863 864
		for (; addr != end; addr += PAGE_SIZE) {
			unsigned long offset;

			offset = (addr & ~PAGEMAP_WALK_MASK) >>
					PAGE_SHIFT;
865 866
			thp_pmd_to_pagemap_entry(&pme, *pmd, offset);
			err = add_to_pagemap(addr, &pme, pm);
867 868
			if (err)
				break;
869 870
		}
		spin_unlock(&walk->mm->page_table_lock);
871
		return err;
872 873
	}

874 875
	if (pmd_trans_unstable(pmd))
		return 0;
876
	for (; addr != end; addr += PAGE_SIZE) {
877 878 879

		/* check to see if we've left 'vma' behind
		 * and need a new, higher one */
880
		if (vma && (addr >= vma->vm_end)) {
881
			vma = find_vma(walk->mm, addr);
882 883
			pme = make_pme(PM_NOT_PRESENT);
		}
884 885 886 887 888 889

		/* check that 'vma' actually covers this address,
		 * and that it isn't a huge page vma */
		if (vma && (vma->vm_start <= addr) &&
		    !is_vm_hugetlb_page(vma)) {
			pte = pte_offset_map(pmd, addr);
890
			pte_to_pagemap_entry(&pme, vma, addr, *pte);
891 892 893
			/* unmap before userspace copy */
			pte_unmap(pte);
		}
894
		err = add_to_pagemap(addr, &pme, pm);
895 896 897 898 899 900 901 902 903
		if (err)
			return err;
	}

	cond_resched();

	return err;
}

904
#ifdef CONFIG_HUGETLB_PAGE
905 906
static void huge_pte_to_pagemap_entry(pagemap_entry_t *pme,
					pte_t pte, int offset)
907 908
{
	if (pte_present(pte))
909 910
		*pme = make_pme(PM_PFRAME(pte_pfn(pte) + offset)
				| PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT);
911 912
	else
		*pme = make_pme(PM_NOT_PRESENT);
913 914
}

915 916 917 918
/* This function walks within one hugetlb entry in the single call */
static int pagemap_hugetlb_range(pte_t *pte, unsigned long hmask,
				 unsigned long addr, unsigned long end,
				 struct mm_walk *walk)
919 920 921
{
	struct pagemapread *pm = walk->private;
	int err = 0;
922
	pagemap_entry_t pme;
923 924

	for (; addr != end; addr += PAGE_SIZE) {
925
		int offset = (addr & ~hmask) >> PAGE_SHIFT;
926 927
		huge_pte_to_pagemap_entry(&pme, *pte, offset);
		err = add_to_pagemap(addr, &pme, pm);
928 929 930 931 932 933 934 935
		if (err)
			return err;
	}

	cond_resched();

	return err;
}
936
#endif /* HUGETLB_PAGE */
937

938 939 940
/*
 * /proc/pid/pagemap - an array mapping virtual pages to pfns
 *
941 942 943
 * For each page in the address space, this file contains one 64-bit entry
 * consisting of the following:
 *
944
 * Bits 0-54  page frame number (PFN) if present
945
 * Bits 0-4   swap type if swapped
946
 * Bits 5-54  swap offset if swapped
947
 * Bits 55-60 page shift (page size = 1<<page shift)
948
 * Bit  61    page is file-page or shared-anon
949 950 951 952 953 954
 * Bit  62    page swapped
 * Bit  63    page present
 *
 * If the page is not present but in swap, then the PFN contains an
 * encoding of the swap file number and the page's offset into the
 * swap. Unmapped pages return a null PFN. This allows determining
955 956 957 958 959 960 961 962 963 964 965 966 967 968
 * precisely which pages are mapped (or in swap) and comparing mapped
 * pages between processes.
 *
 * Efficient users of this interface will use /proc/pid/maps to
 * determine which areas of memory are actually mapped and llseek to
 * skip over unmapped regions.
 */
static ssize_t pagemap_read(struct file *file, char __user *buf,
			    size_t count, loff_t *ppos)
{
	struct task_struct *task = get_proc_task(file->f_path.dentry->d_inode);
	struct mm_struct *mm;
	struct pagemapread pm;
	int ret = -ESRCH;
969
	struct mm_walk pagemap_walk = {};
970 971 972 973
	unsigned long src;
	unsigned long svpfn;
	unsigned long start_vaddr;
	unsigned long end_vaddr;
974
	int copied = 0;
975 976 977 978 979 980

	if (!task)
		goto out;

	ret = -EINVAL;
	/* file position must be aligned */
981
	if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
982
		goto out_task;
983 984

	ret = 0;
985 986 987
	if (!count)
		goto out_task;

988 989
	pm.len = PM_ENTRY_BYTES * (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
	pm.buffer = kmalloc(pm.len, GFP_TEMPORARY);
990
	ret = -ENOMEM;
991
	if (!pm.buffer)
992 993
		goto out_task;

C
Cong Wang 已提交
994
	mm = mm_access(task, PTRACE_MODE_READ);
995 996 997
	ret = PTR_ERR(mm);
	if (!mm || IS_ERR(mm))
		goto out_free;
998

999 1000
	pagemap_walk.pmd_entry = pagemap_pte_range;
	pagemap_walk.pte_hole = pagemap_pte_hole;
1001
#ifdef CONFIG_HUGETLB_PAGE
1002
	pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1003
#endif
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	pagemap_walk.mm = mm;
	pagemap_walk.private = &pm;

	src = *ppos;
	svpfn = src / PM_ENTRY_BYTES;
	start_vaddr = svpfn << PAGE_SHIFT;
	end_vaddr = TASK_SIZE_OF(task);

	/* watch out for wraparound */
	if (svpfn > TASK_SIZE_OF(task) >> PAGE_SHIFT)
		start_vaddr = end_vaddr;

	/*
	 * The odds are that this will stop walking way
	 * before end_vaddr, because the length of the
	 * user buffer is tracked in "pm", and the walk
	 * will stop when we hit the end of the buffer.
	 */
1022 1023 1024 1025 1026 1027
	ret = 0;
	while (count && (start_vaddr < end_vaddr)) {
		int len;
		unsigned long end;

		pm.pos = 0;
1028
		end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1029 1030 1031 1032 1033 1034 1035 1036 1037
		/* overflow ? */
		if (end < start_vaddr || end > end_vaddr)
			end = end_vaddr;
		down_read(&mm->mmap_sem);
		ret = walk_page_range(start_vaddr, end, &pagemap_walk);
		up_read(&mm->mmap_sem);
		start_vaddr = end;

		len = min(count, PM_ENTRY_BYTES * pm.pos);
1038
		if (copy_to_user(buf, pm.buffer, len)) {
1039
			ret = -EFAULT;
1040
			goto out_mm;
1041 1042 1043 1044
		}
		copied += len;
		buf += len;
		count -= len;
1045
	}
1046 1047 1048 1049
	*ppos += copied;
	if (!ret || ret == PM_END_OF_BUFFER)
		ret = copied;

1050 1051
out_mm:
	mmput(mm);
1052 1053
out_free:
	kfree(pm.buffer);
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
out_task:
	put_task_struct(task);
out:
	return ret;
}

const struct file_operations proc_pagemap_operations = {
	.llseek		= mem_lseek, /* borrow this */
	.read		= pagemap_read,
};
1064
#endif /* CONFIG_PROC_PAGE_MONITOR */
1065

1066 1067
#ifdef CONFIG_NUMA

1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
struct numa_maps {
	struct vm_area_struct *vma;
	unsigned long pages;
	unsigned long anon;
	unsigned long active;
	unsigned long writeback;
	unsigned long mapcount_max;
	unsigned long dirty;
	unsigned long swapcache;
	unsigned long node[MAX_NUMNODES];
};

1080 1081 1082 1083 1084
struct numa_maps_private {
	struct proc_maps_private proc_maps;
	struct numa_maps md;
};

1085 1086
static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
			unsigned long nr_pages)
1087 1088 1089
{
	int count = page_mapcount(page);

1090
	md->pages += nr_pages;
1091
	if (pte_dirty || PageDirty(page))
1092
		md->dirty += nr_pages;
1093 1094

	if (PageSwapCache(page))
1095
		md->swapcache += nr_pages;
1096 1097

	if (PageActive(page) || PageUnevictable(page))
1098
		md->active += nr_pages;
1099 1100

	if (PageWriteback(page))
1101
		md->writeback += nr_pages;
1102 1103

	if (PageAnon(page))
1104
		md->anon += nr_pages;
1105 1106 1107 1108

	if (count > md->mapcount_max)
		md->mapcount_max = count;

1109
	md->node[page_to_nid(page)] += nr_pages;
1110 1111
}

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
		unsigned long addr)
{
	struct page *page;
	int nid;

	if (!pte_present(pte))
		return NULL;

	page = vm_normal_page(vma, addr, pte);
	if (!page)
		return NULL;

	if (PageReserved(page))
		return NULL;

	nid = page_to_nid(page);
	if (!node_isset(nid, node_states[N_HIGH_MEMORY]))
		return NULL;

	return page;
}

1135 1136 1137 1138 1139 1140 1141 1142 1143
static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
		unsigned long end, struct mm_walk *walk)
{
	struct numa_maps *md;
	spinlock_t *ptl;
	pte_t *orig_pte;
	pte_t *pte;

	md = walk->private;
1144 1145 1146 1147 1148 1149 1150 1151 1152

	if (pmd_trans_huge_lock(pmd, md->vma) == 1) {
		pte_t huge_pte = *(pte_t *)pmd;
		struct page *page;

		page = can_gather_numa_stats(huge_pte, md->vma, addr);
		if (page)
			gather_stats(page, md, pte_dirty(huge_pte),
				     HPAGE_PMD_SIZE/PAGE_SIZE);
1153
		spin_unlock(&walk->mm->page_table_lock);
1154
		return 0;
1155 1156
	}

1157 1158
	if (pmd_trans_unstable(pmd))
		return 0;
1159 1160
	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
	do {
1161
		struct page *page = can_gather_numa_stats(*pte, md->vma, addr);
1162 1163
		if (!page)
			continue;
1164
		gather_stats(page, md, pte_dirty(*pte), 1);
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184

	} while (pte++, addr += PAGE_SIZE, addr != end);
	pte_unmap_unlock(orig_pte, ptl);
	return 0;
}
#ifdef CONFIG_HUGETLB_PAGE
static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
		unsigned long addr, unsigned long end, struct mm_walk *walk)
{
	struct numa_maps *md;
	struct page *page;

	if (pte_none(*pte))
		return 0;

	page = pte_page(*pte);
	if (!page)
		return 0;

	md = walk->private;
1185
	gather_stats(page, md, pte_dirty(*pte), 1);
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
	return 0;
}

#else
static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
		unsigned long addr, unsigned long end, struct mm_walk *walk)
{
	return 0;
}
#endif

/*
 * Display pages allocated per node and memory policy via /proc.
 */
1200
static int show_numa_map(struct seq_file *m, void *v, int is_pid)
1201
{
1202 1203
	struct numa_maps_private *numa_priv = m->private;
	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1204
	struct vm_area_struct *vma = v;
1205
	struct numa_maps *md = &numa_priv->md;
1206
	struct file *file = vma->vm_file;
1207
	struct task_struct *task = proc_priv->task;
1208 1209 1210 1211 1212 1213 1214 1215 1216
	struct mm_struct *mm = vma->vm_mm;
	struct mm_walk walk = {};
	struct mempolicy *pol;
	int n;
	char buffer[50];

	if (!mm)
		return 0;

1217 1218
	/* Ensure we start with an empty set of numa_maps statistics. */
	memset(md, 0, sizeof(*md));
1219 1220 1221 1222 1223 1224 1225 1226

	md->vma = vma;

	walk.hugetlb_entry = gather_hugetbl_stats;
	walk.pmd_entry = gather_pte_stats;
	walk.private = md;
	walk.mm = mm;

1227
	pol = get_vma_policy(task, vma, vma->vm_start);
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
	mpol_to_str(buffer, sizeof(buffer), pol, 0);
	mpol_cond_put(pol);

	seq_printf(m, "%08lx %s", vma->vm_start, buffer);

	if (file) {
		seq_printf(m, " file=");
		seq_path(m, &file->f_path, "\n\t= ");
	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
		seq_printf(m, " heap");
1238
	} else {
1239
		pid_t tid = vm_is_stack(task, vma, is_pid);
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
		if (tid != 0) {
			/*
			 * Thread stack in /proc/PID/task/TID/maps or
			 * the main process stack.
			 */
			if (!is_pid || (vma->vm_start <= mm->start_stack &&
			    vma->vm_end >= mm->start_stack))
				seq_printf(m, " stack");
			else
				seq_printf(m, " stack:%d", tid);
		}
1251 1252
	}

1253 1254 1255
	if (is_vm_hugetlb_page(vma))
		seq_printf(m, " huge");

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	walk_page_range(vma->vm_start, vma->vm_end, &walk);

	if (!md->pages)
		goto out;

	if (md->anon)
		seq_printf(m, " anon=%lu", md->anon);

	if (md->dirty)
		seq_printf(m, " dirty=%lu", md->dirty);

	if (md->pages != md->anon && md->pages != md->dirty)
		seq_printf(m, " mapped=%lu", md->pages);

	if (md->mapcount_max > 1)
		seq_printf(m, " mapmax=%lu", md->mapcount_max);

	if (md->swapcache)
		seq_printf(m, " swapcache=%lu", md->swapcache);

	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
		seq_printf(m, " active=%lu", md->active);

	if (md->writeback)
		seq_printf(m, " writeback=%lu", md->writeback);

	for_each_node_state(n, N_HIGH_MEMORY)
		if (md->node[n])
			seq_printf(m, " N%d=%lu", n, md->node[n]);
out:
	seq_putc(m, '\n');

	if (m->count < m->size)
1289
		m->version = (vma != proc_priv->tail_vma) ? vma->vm_start : 0;
1290 1291
	return 0;
}
1292

1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
static int show_pid_numa_map(struct seq_file *m, void *v)
{
	return show_numa_map(m, v, 1);
}

static int show_tid_numa_map(struct seq_file *m, void *v)
{
	return show_numa_map(m, v, 0);
}

1303
static const struct seq_operations proc_pid_numa_maps_op = {
1304 1305 1306 1307
	.start  = m_start,
	.next   = m_next,
	.stop   = m_stop,
	.show   = show_pid_numa_map,
1308
};
1309

1310 1311 1312 1313 1314 1315 1316 1317 1318
static const struct seq_operations proc_tid_numa_maps_op = {
	.start  = m_start,
	.next   = m_next,
	.stop   = m_stop,
	.show   = show_tid_numa_map,
};

static int numa_maps_open(struct inode *inode, struct file *file,
			  const struct seq_operations *ops)
1319
{
1320 1321 1322 1323 1324
	struct numa_maps_private *priv;
	int ret = -ENOMEM;
	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	if (priv) {
		priv->proc_maps.pid = proc_pid(inode);
1325
		ret = seq_open(file, ops);
1326 1327 1328 1329 1330 1331 1332 1333
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = priv;
		} else {
			kfree(priv);
		}
	}
	return ret;
1334 1335
}

1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
static int pid_numa_maps_open(struct inode *inode, struct file *file)
{
	return numa_maps_open(inode, file, &proc_pid_numa_maps_op);
}

static int tid_numa_maps_open(struct inode *inode, struct file *file)
{
	return numa_maps_open(inode, file, &proc_tid_numa_maps_op);
}

const struct file_operations proc_pid_numa_maps_operations = {
	.open		= pid_numa_maps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};

const struct file_operations proc_tid_numa_maps_operations = {
	.open		= tid_numa_maps_open,
1355 1356
	.read		= seq_read,
	.llseek		= seq_lseek,
1357
	.release	= seq_release_private,
1358
};
1359
#endif /* CONFIG_NUMA */