task_mmu.c 12.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/mount.h>
#include <linux/seq_file.h>
M
Mauricio Lin 已提交
5
#include <linux/highmem.h>
K
Kees Cook 已提交
6
#include <linux/ptrace.h>
7 8
#include <linux/pagemap.h>
#include <linux/mempolicy.h>
M
Mauricio Lin 已提交
9

L
Linus Torvalds 已提交
10 11
#include <asm/elf.h>
#include <asm/uaccess.h>
M
Mauricio Lin 已提交
12
#include <asm/tlbflush.h>
L
Linus Torvalds 已提交
13 14 15 16 17
#include "internal.h"

char *task_mem(struct mm_struct *mm, char *buffer)
{
	unsigned long data, text, lib;
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;

	/*
	 * Note: to minimize their overhead, mm maintains hiwater_vm and
	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
	 * collector of these hiwater stats must therefore get total_vm
	 * and rss too, which will usually be the higher.  Barriers? not
	 * worth the effort, such snapshots can always be inconsistent.
	 */
	hiwater_vm = total_vm = mm->total_vm;
	if (hiwater_vm < mm->hiwater_vm)
		hiwater_vm = mm->hiwater_vm;
	hiwater_rss = total_rss = get_mm_rss(mm);
	if (hiwater_rss < mm->hiwater_rss)
		hiwater_rss = mm->hiwater_rss;
L
Linus Torvalds 已提交
33 34 35 36 37

	data = mm->total_vm - mm->shared_vm - mm->stack_vm;
	text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
	lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
	buffer += sprintf(buffer,
38
		"VmPeak:\t%8lu kB\n"
L
Linus Torvalds 已提交
39 40
		"VmSize:\t%8lu kB\n"
		"VmLck:\t%8lu kB\n"
41
		"VmHWM:\t%8lu kB\n"
L
Linus Torvalds 已提交
42 43 44 45 46 47
		"VmRSS:\t%8lu kB\n"
		"VmData:\t%8lu kB\n"
		"VmStk:\t%8lu kB\n"
		"VmExe:\t%8lu kB\n"
		"VmLib:\t%8lu kB\n"
		"VmPTE:\t%8lu kB\n",
48 49
		hiwater_vm << (PAGE_SHIFT-10),
		(total_vm - mm->reserved_vm) << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
50
		mm->locked_vm << (PAGE_SHIFT-10),
51 52
		hiwater_rss << (PAGE_SHIFT-10),
		total_rss << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
53 54 55 56 57 58 59 60 61 62 63 64 65 66
		data << (PAGE_SHIFT-10),
		mm->stack_vm << (PAGE_SHIFT-10), text, lib,
		(PTRS_PER_PTE*sizeof(pte_t)*mm->nr_ptes) >> 10);
	return buffer;
}

unsigned long task_vsize(struct mm_struct *mm)
{
	return PAGE_SIZE * mm->total_vm;
}

int task_statm(struct mm_struct *mm, int *shared, int *text,
	       int *data, int *resident)
{
67
	*shared = get_mm_counter(mm, file_rss);
L
Linus Torvalds 已提交
68 69 70
	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
								>> PAGE_SHIFT;
	*data = mm->total_vm - mm->shared_vm;
71
	*resident = *shared + get_mm_counter(mm, anon_rss);
L
Linus Torvalds 已提交
72 73 74 75 76 77 78
	return mm->total_vm;
}

int proc_exe_link(struct inode *inode, struct dentry **dentry, struct vfsmount **mnt)
{
	struct vm_area_struct * vma;
	int result = -ENOENT;
79 80
	struct task_struct *task = get_proc_task(inode);
	struct mm_struct * mm = NULL;
L
Linus Torvalds 已提交
81

82 83 84 85
	if (task) {
		mm = get_task_mm(task);
		put_task_struct(task);
	}
L
Linus Torvalds 已提交
86 87 88 89 90 91 92 93 94 95 96 97
	if (!mm)
		goto out;
	down_read(&mm->mmap_sem);

	vma = mm->mmap;
	while (vma) {
		if ((vma->vm_flags & VM_EXECUTABLE) && vma->vm_file)
			break;
		vma = vma->vm_next;
	}

	if (vma) {
98 99
		*mnt = mntget(vma->vm_file->f_path.mnt);
		*dentry = dget(vma->vm_file->f_path.dentry);
L
Linus Torvalds 已提交
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
		result = 0;
	}

	up_read(&mm->mmap_sem);
	mmput(mm);
out:
	return result;
}

static void pad_len_spaces(struct seq_file *m, int len)
{
	len = 25 + sizeof(void*) * 6 - len;
	if (len < 1)
		len = 1;
	seq_printf(m, "%*c", len, ' ');
}

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
/*
 * Proportional Set Size(PSS): my share of RSS.
 *
 * PSS of a process is the count of pages it has in memory, where each
 * page is divided by the number of processes sharing it.  So if a
 * process has 1000 pages all to itself, and 1000 shared with one other
 * process, its PSS will be 1500.
 *
 * To keep (accumulated) division errors low, we adopt a 64bit
 * fixed-point pss counter to minimize division errors. So (pss >>
 * PSS_SHIFT) would be the real byte count.
 *
 * A shift of 12 before division means (assuming 4K page size):
 * 	- 1M 3-user-pages add up to 8KB errors;
 * 	- supports mapcount up to 2^24, or 16M;
 * 	- supports PSS up to 2^52 bytes, or 4PB.
 */
#define PSS_SHIFT 12

M
Mauricio Lin 已提交
136 137
struct mem_size_stats
{
138
	struct vm_area_struct *vma;
M
Mauricio Lin 已提交
139 140 141 142 143
	unsigned long resident;
	unsigned long shared_clean;
	unsigned long shared_dirty;
	unsigned long private_clean;
	unsigned long private_dirty;
144
	unsigned long referenced;
145
	u64 pss;
M
Mauricio Lin 已提交
146 147 148
};

static int show_map_internal(struct seq_file *m, void *v, struct mem_size_stats *mss)
L
Linus Torvalds 已提交
149
{
150 151
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;
M
Mauricio Lin 已提交
152 153 154 155
	struct vm_area_struct *vma = v;
	struct mm_struct *mm = vma->vm_mm;
	struct file *file = vma->vm_file;
	int flags = vma->vm_flags;
L
Linus Torvalds 已提交
156 157 158 159
	unsigned long ino = 0;
	dev_t dev = 0;
	int len;

K
Kees Cook 已提交
160 161 162
	if (maps_protect && !ptrace_may_attach(task))
		return -EACCES;

L
Linus Torvalds 已提交
163
	if (file) {
164
		struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
165 166 167 168 169
		dev = inode->i_sb->s_dev;
		ino = inode->i_ino;
	}

	seq_printf(m, "%08lx-%08lx %c%c%c%c %08lx %02x:%02x %lu %n",
M
Mauricio Lin 已提交
170 171
			vma->vm_start,
			vma->vm_end,
L
Linus Torvalds 已提交
172 173 174 175
			flags & VM_READ ? 'r' : '-',
			flags & VM_WRITE ? 'w' : '-',
			flags & VM_EXEC ? 'x' : '-',
			flags & VM_MAYSHARE ? 's' : 'p',
M
Mauricio Lin 已提交
176
			vma->vm_pgoff << PAGE_SHIFT,
L
Linus Torvalds 已提交
177 178 179 180 181 182
			MAJOR(dev), MINOR(dev), ino, &len);

	/*
	 * Print the dentry name for named mappings, and a
	 * special [heap] marker for the heap:
	 */
M
Mauricio Lin 已提交
183
	if (file) {
L
Linus Torvalds 已提交
184
		pad_len_spaces(m, len);
185
		seq_path(m, file->f_path.mnt, file->f_path.dentry, "\n");
L
Linus Torvalds 已提交
186
	} else {
187 188 189 190
		const char *name = arch_vma_name(vma);
		if (!name) {
			if (mm) {
				if (vma->vm_start <= mm->start_brk &&
M
Mauricio Lin 已提交
191
						vma->vm_end >= mm->brk) {
192 193 194 195
					name = "[heap]";
				} else if (vma->vm_start <= mm->start_stack &&
					   vma->vm_end >= mm->start_stack) {
					name = "[stack]";
L
Linus Torvalds 已提交
196
				}
197 198
			} else {
				name = "[vdso]";
L
Linus Torvalds 已提交
199
			}
200 201
		}
		if (name) {
L
Linus Torvalds 已提交
202
			pad_len_spaces(m, len);
203
			seq_puts(m, name);
L
Linus Torvalds 已提交
204 205 206
		}
	}
	seq_putc(m, '\n');
M
Mauricio Lin 已提交
207 208 209

	if (mss)
		seq_printf(m,
210 211
			   "Size:           %8lu kB\n"
			   "Rss:            %8lu kB\n"
212
			   "Pss:            %8lu kB\n"
213 214 215 216
			   "Shared_Clean:   %8lu kB\n"
			   "Shared_Dirty:   %8lu kB\n"
			   "Private_Clean:  %8lu kB\n"
			   "Private_Dirty:  %8lu kB\n"
217
			   "Referenced:     %8lu kB\n",
M
Mauricio Lin 已提交
218 219
			   (vma->vm_end - vma->vm_start) >> 10,
			   mss->resident >> 10,
220
			   (unsigned long)(mss->pss >> (10 + PSS_SHIFT)),
M
Mauricio Lin 已提交
221 222 223
			   mss->shared_clean  >> 10,
			   mss->shared_dirty  >> 10,
			   mss->private_clean >> 10,
224 225
			   mss->private_dirty >> 10,
			   mss->referenced >> 10);
M
Mauricio Lin 已提交
226 227 228

	if (m->count < m->size)  /* vma is copied successfully */
		m->version = (vma != get_gate_vma(task))? vma->vm_start: 0;
L
Linus Torvalds 已提交
229 230 231
	return 0;
}

M
Mauricio Lin 已提交
232 233
static int show_map(struct seq_file *m, void *v)
{
234
	return show_map_internal(m, v, NULL);
M
Mauricio Lin 已提交
235 236
}

237 238
static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
			   void *private)
M
Mauricio Lin 已提交
239
{
240
	struct mem_size_stats *mss = private;
241
	struct vm_area_struct *vma = mss->vma;
M
Mauricio Lin 已提交
242
	pte_t *pte, ptent;
243
	spinlock_t *ptl;
M
Mauricio Lin 已提交
244
	struct page *page;
245
	int mapcount;
M
Mauricio Lin 已提交
246

247
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
248
	for (; addr != end; pte++, addr += PAGE_SIZE) {
M
Mauricio Lin 已提交
249
		ptent = *pte;
250
		if (!pte_present(ptent))
M
Mauricio Lin 已提交
251 252 253
			continue;

		mss->resident += PAGE_SIZE;
N
Nick Piggin 已提交
254 255 256

		page = vm_normal_page(vma, addr, ptent);
		if (!page)
M
Mauricio Lin 已提交
257 258
			continue;

259 260 261
		/* Accumulate the size in pages that have been accessed. */
		if (pte_young(ptent) || PageReferenced(page))
			mss->referenced += PAGE_SIZE;
262 263
		mapcount = page_mapcount(page);
		if (mapcount >= 2) {
M
Mauricio Lin 已提交
264 265 266 267
			if (pte_dirty(ptent))
				mss->shared_dirty += PAGE_SIZE;
			else
				mss->shared_clean += PAGE_SIZE;
268
			mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
M
Mauricio Lin 已提交
269 270 271 272 273
		} else {
			if (pte_dirty(ptent))
				mss->private_dirty += PAGE_SIZE;
			else
				mss->private_clean += PAGE_SIZE;
274
			mss->pss += (PAGE_SIZE << PSS_SHIFT);
M
Mauricio Lin 已提交
275
		}
276
	}
277 278
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
279
	return 0;
M
Mauricio Lin 已提交
280 281
}

282 283
static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
				unsigned long end, void *private)
284
{
285
	struct vm_area_struct *vma = private;
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
	pte_t *pte, ptent;
	spinlock_t *ptl;
	struct page *page;

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE) {
		ptent = *pte;
		if (!pte_present(ptent))
			continue;

		page = vm_normal_page(vma, addr, ptent);
		if (!page)
			continue;

		/* Clear accessed and referenced bits. */
		ptep_test_and_clear_young(vma, addr, pte);
		ClearPageReferenced(page);
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
306
	return 0;
307 308
}

309
static struct mm_walk smaps_walk = { .pmd_entry = smaps_pte_range };
M
Mauricio Lin 已提交
310 311 312 313 314 315 316

static int show_smap(struct seq_file *m, void *v)
{
	struct vm_area_struct *vma = v;
	struct mem_size_stats mss;

	memset(&mss, 0, sizeof mss);
317
	mss.vma = vma;
N
Nick Piggin 已提交
318
	if (vma->vm_mm && !is_vm_hugetlb_page(vma))
319 320
		walk_page_range(vma->vm_mm, vma->vm_start, vma->vm_end,
				&smaps_walk, &mss);
M
Mauricio Lin 已提交
321 322 323
	return show_map_internal(m, v, &mss);
}

324 325
static struct mm_walk clear_refs_walk = { .pmd_entry = clear_refs_pte_range };

326 327 328 329 330 331 332
void clear_refs_smap(struct mm_struct *mm)
{
	struct vm_area_struct *vma;

	down_read(&mm->mmap_sem);
	for (vma = mm->mmap; vma; vma = vma->vm_next)
		if (vma->vm_mm && !is_vm_hugetlb_page(vma))
333 334
			walk_page_range(vma->vm_mm, vma->vm_start, vma->vm_end,
					&clear_refs_walk, vma);
335 336 337 338
	flush_tlb_mm(mm);
	up_read(&mm->mmap_sem);
}

L
Linus Torvalds 已提交
339 340
static void *m_start(struct seq_file *m, loff_t *pos)
{
341
	struct proc_maps_private *priv = m->private;
L
Linus Torvalds 已提交
342 343
	unsigned long last_addr = m->version;
	struct mm_struct *mm;
344
	struct vm_area_struct *vma, *tail_vma = NULL;
L
Linus Torvalds 已提交
345 346
	loff_t l = *pos;

347 348 349 350
	/* Clear the per syscall fields in priv */
	priv->task = NULL;
	priv->tail_vma = NULL;

L
Linus Torvalds 已提交
351 352 353
	/*
	 * We remember last_addr rather than next_addr to hit with
	 * mmap_cache most of the time. We have zero last_addr at
M
Mauricio Lin 已提交
354 355
	 * the beginning and also after lseek. We will have -1 last_addr
	 * after the end of the vmas.
L
Linus Torvalds 已提交
356 357 358 359 360
	 */

	if (last_addr == -1UL)
		return NULL;

361
	priv->task = get_pid_task(priv->pid, PIDTYPE_PID);
362 363 364
	if (!priv->task)
		return NULL;

365
	mm = mm_for_maps(priv->task);
L
Linus Torvalds 已提交
366 367 368
	if (!mm)
		return NULL;

369
	priv->tail_vma = tail_vma = get_gate_vma(priv->task);
L
Linus Torvalds 已提交
370 371

	/* Start with last addr hint */
M
Mauricio Lin 已提交
372 373
	if (last_addr && (vma = find_vma(mm, last_addr))) {
		vma = vma->vm_next;
L
Linus Torvalds 已提交
374 375 376 377
		goto out;
	}

	/*
M
Mauricio Lin 已提交
378
	 * Check the vma index is within the range and do
L
Linus Torvalds 已提交
379 380
	 * sequential scan until m_index.
	 */
M
Mauricio Lin 已提交
381
	vma = NULL;
L
Linus Torvalds 已提交
382
	if ((unsigned long)l < mm->map_count) {
M
Mauricio Lin 已提交
383 384 385
		vma = mm->mmap;
		while (l-- && vma)
			vma = vma->vm_next;
L
Linus Torvalds 已提交
386 387 388 389
		goto out;
	}

	if (l != mm->map_count)
M
Mauricio Lin 已提交
390
		tail_vma = NULL; /* After gate vma */
L
Linus Torvalds 已提交
391 392

out:
M
Mauricio Lin 已提交
393 394
	if (vma)
		return vma;
L
Linus Torvalds 已提交
395

M
Mauricio Lin 已提交
396 397
	/* End of vmas has been reached */
	m->version = (tail_vma != NULL)? 0: -1UL;
L
Linus Torvalds 已提交
398 399
	up_read(&mm->mmap_sem);
	mmput(mm);
M
Mauricio Lin 已提交
400
	return tail_vma;
L
Linus Torvalds 已提交
401 402
}

403
static void vma_stop(struct proc_maps_private *priv, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
404
{
405
	if (vma && vma != priv->tail_vma) {
M
Mauricio Lin 已提交
406
		struct mm_struct *mm = vma->vm_mm;
L
Linus Torvalds 已提交
407 408 409 410 411 412 413
		up_read(&mm->mmap_sem);
		mmput(mm);
	}
}

static void *m_next(struct seq_file *m, void *v, loff_t *pos)
{
414
	struct proc_maps_private *priv = m->private;
M
Mauricio Lin 已提交
415
	struct vm_area_struct *vma = v;
416
	struct vm_area_struct *tail_vma = priv->tail_vma;
L
Linus Torvalds 已提交
417 418

	(*pos)++;
M
Mauricio Lin 已提交
419 420
	if (vma && (vma != tail_vma) && vma->vm_next)
		return vma->vm_next;
421
	vma_stop(priv, vma);
M
Mauricio Lin 已提交
422
	return (vma != tail_vma)? tail_vma: NULL;
L
Linus Torvalds 已提交
423 424
}

425 426 427 428 429 430 431 432 433 434
static void m_stop(struct seq_file *m, void *v)
{
	struct proc_maps_private *priv = m->private;
	struct vm_area_struct *vma = v;

	vma_stop(priv, vma);
	if (priv->task)
		put_task_struct(priv->task);
}

435
static struct seq_operations proc_pid_maps_op = {
L
Linus Torvalds 已提交
436 437 438 439 440
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= show_map
};
441

442
static struct seq_operations proc_pid_smaps_op = {
M
Mauricio Lin 已提交
443 444 445 446 447 448
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= show_smap
};

449 450 451
static int do_maps_open(struct inode *inode, struct file *file,
			struct seq_operations *ops)
{
452 453 454 455
	struct proc_maps_private *priv;
	int ret = -ENOMEM;
	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	if (priv) {
456
		priv->pid = proc_pid(inode);
457 458 459 460 461 462 463
		ret = seq_open(file, ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = priv;
		} else {
			kfree(priv);
		}
464 465 466 467 468 469 470 471 472
	}
	return ret;
}

static int maps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_pid_maps_op);
}

473
const struct file_operations proc_maps_operations = {
474 475 476
	.open		= maps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
477
	.release	= seq_release_private,
478 479
};

480
#ifdef CONFIG_NUMA
481
extern int show_numa_map(struct seq_file *m, void *v);
482

K
Kees Cook 已提交
483 484 485 486 487 488 489 490 491 492 493
static int show_numa_map_checked(struct seq_file *m, void *v)
{
	struct proc_maps_private *priv = m->private;
	struct task_struct *task = priv->task;

	if (maps_protect && !ptrace_may_attach(task))
		return -EACCES;

	return show_numa_map(m, v);
}

494
static struct seq_operations proc_pid_numa_maps_op = {
495 496 497
        .start  = m_start,
        .next   = m_next,
        .stop   = m_stop,
K
Kees Cook 已提交
498
        .show   = show_numa_map_checked
499
};
500 501 502 503 504 505

static int numa_maps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_pid_numa_maps_op);
}

506
const struct file_operations proc_numa_maps_operations = {
507 508 509
	.open		= numa_maps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
510
	.release	= seq_release_private,
511
};
512
#endif
513 514 515 516 517 518

static int smaps_open(struct inode *inode, struct file *file)
{
	return do_maps_open(inode, file, &proc_pid_smaps_op);
}

519
const struct file_operations proc_smaps_operations = {
520 521 522
	.open		= smaps_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
523
	.release	= seq_release_private,
524
};