buffer_sync.c 13.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/**
 * @file buffer_sync.c
 *
 * @remark Copyright 2002 OProfile authors
 * @remark Read the file COPYING
 *
 * @author John Levon <levon@movementarian.org>
8
 * @author Barry Kasindorf
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
 *
 * This is the core of the buffer management. Each
 * CPU buffer is processed and entered into the
 * global event buffer. Such processing is necessary
 * in several circumstances, mentioned below.
 *
 * The processing does the job of converting the
 * transitory EIP value into a persistent dentry/offset
 * value that the profiler can record at its leisure.
 *
 * See fs/dcookies.c for a description of the dentry/offset
 * objects.
 */

#include <linux/mm.h>
#include <linux/workqueue.h>
#include <linux/notifier.h>
#include <linux/dcookies.h>
#include <linux/profile.h>
#include <linux/module.h>
#include <linux/fs.h>
30
#include <linux/oprofile.h>
A
Alexey Dobriyan 已提交
31
#include <linux/sched.h>
32

L
Linus Torvalds 已提交
33 34 35 36
#include "oprofile_stats.h"
#include "event_buffer.h"
#include "cpu_buffer.h"
#include "buffer_sync.h"
37

L
Linus Torvalds 已提交
38 39 40 41 42 43 44 45 46 47
static LIST_HEAD(dying_tasks);
static LIST_HEAD(dead_tasks);
static cpumask_t marked_cpus = CPU_MASK_NONE;
static DEFINE_SPINLOCK(task_mortuary);
static void process_task_mortuary(void);

/* Take ownership of the task struct and place it on the
 * list for processing. Only after two full buffer syncs
 * does the task eventually get freed, because by then
 * we are sure we will not reference it again.
48 49
 * Can be invoked from softirq via RCU callback due to
 * call_rcu() of the task struct, hence the _irqsave.
L
Linus Torvalds 已提交
50
 */
51 52
static int
task_free_notify(struct notifier_block *self, unsigned long val, void *data)
L
Linus Torvalds 已提交
53
{
54
	unsigned long flags;
55
	struct task_struct *task = data;
56
	spin_lock_irqsave(&task_mortuary, flags);
L
Linus Torvalds 已提交
57
	list_add(&task->tasks, &dying_tasks);
58
	spin_unlock_irqrestore(&task_mortuary, flags);
L
Linus Torvalds 已提交
59 60 61 62 63 64 65
	return NOTIFY_OK;
}


/* The task is on its way out. A sync of the buffer means we can catch
 * any remaining samples for this task.
 */
66 67
static int
task_exit_notify(struct notifier_block *self, unsigned long val, void *data)
L
Linus Torvalds 已提交
68 69 70 71
{
	/* To avoid latency problems, we only process the current CPU,
	 * hoping that most samples for the task are on this CPU
	 */
I
Ingo Molnar 已提交
72
	sync_buffer(raw_smp_processor_id());
73
	return 0;
L
Linus Torvalds 已提交
74 75 76 77 78 79 80 81
}


/* The task is about to try a do_munmap(). We peek at what it's going to
 * do, and if it's an executable region, process the samples first, so
 * we don't lose any. This does not have to be exact, it's a QoI issue
 * only.
 */
82 83
static int
munmap_notify(struct notifier_block *self, unsigned long val, void *data)
L
Linus Torvalds 已提交
84 85
{
	unsigned long addr = (unsigned long)data;
86 87
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *mpnt;
L
Linus Torvalds 已提交
88 89 90 91 92 93 94 95 96

	down_read(&mm->mmap_sem);

	mpnt = find_vma(mm, addr);
	if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) {
		up_read(&mm->mmap_sem);
		/* To avoid latency problems, we only process the current CPU,
		 * hoping that most samples for the task are on this CPU
		 */
I
Ingo Molnar 已提交
97
		sync_buffer(raw_smp_processor_id());
L
Linus Torvalds 已提交
98 99 100 101 102 103 104
		return 0;
	}

	up_read(&mm->mmap_sem);
	return 0;
}

105

L
Linus Torvalds 已提交
106 107 108
/* We need to be told about new modules so we don't attribute to a previously
 * loaded module, or drop the samples on the floor.
 */
109 110
static int
module_load_notify(struct notifier_block *self, unsigned long val, void *data)
L
Linus Torvalds 已提交
111 112 113 114 115 116
{
#ifdef CONFIG_MODULES
	if (val != MODULE_STATE_COMING)
		return 0;

	/* FIXME: should we process all CPU buffers ? */
117
	mutex_lock(&buffer_mutex);
L
Linus Torvalds 已提交
118 119
	add_event_entry(ESCAPE_CODE);
	add_event_entry(MODULE_LOADED_CODE);
120
	mutex_unlock(&buffer_mutex);
L
Linus Torvalds 已提交
121 122 123 124
#endif
	return 0;
}

125

L
Linus Torvalds 已提交
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
static struct notifier_block task_free_nb = {
	.notifier_call	= task_free_notify,
};

static struct notifier_block task_exit_nb = {
	.notifier_call	= task_exit_notify,
};

static struct notifier_block munmap_nb = {
	.notifier_call	= munmap_notify,
};

static struct notifier_block module_load_nb = {
	.notifier_call = module_load_notify,
};

142

L
Linus Torvalds 已提交
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
static void end_sync(void)
{
	end_cpu_work();
	/* make sure we don't leak task structs */
	process_task_mortuary();
	process_task_mortuary();
}


int sync_start(void)
{
	int err;

	start_cpu_work();

	err = task_handoff_register(&task_free_nb);
	if (err)
		goto out1;
	err = profile_event_register(PROFILE_TASK_EXIT, &task_exit_nb);
	if (err)
		goto out2;
	err = profile_event_register(PROFILE_MUNMAP, &munmap_nb);
	if (err)
		goto out3;
	err = register_module_notifier(&module_load_nb);
	if (err)
		goto out4;

out:
	return err;
out4:
	profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
out3:
	profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
out2:
	task_handoff_unregister(&task_free_nb);
out1:
	end_sync();
	goto out;
}


void sync_stop(void)
{
	unregister_module_notifier(&module_load_nb);
	profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
	profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
	task_handoff_unregister(&task_free_nb);
	end_sync();
}

194

L
Linus Torvalds 已提交
195 196 197 198
/* Optimisation. We can manage without taking the dcookie sem
 * because we cannot reach this code without at least one
 * dcookie user still being registered (namely, the reader
 * of the event buffer). */
199
static inline unsigned long fast_get_dcookie(struct path *path)
L
Linus Torvalds 已提交
200 201
{
	unsigned long cookie;
202 203 204 205

	if (path->dentry->d_cookie)
		return (unsigned long)path->dentry;
	get_dcookie(path, &cookie);
L
Linus Torvalds 已提交
206 207 208
	return cookie;
}

209

L
Linus Torvalds 已提交
210 211 212 213 214
/* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
 * which corresponds loosely to "application name". This is
 * not strictly necessary but allows oprofile to associate
 * shared-library samples with particular applications
 */
215
static unsigned long get_exec_dcookie(struct mm_struct *mm)
L
Linus Torvalds 已提交
216
{
217
	unsigned long cookie = NO_COOKIE;
218 219
	struct vm_area_struct *vma;

L
Linus Torvalds 已提交
220 221
	if (!mm)
		goto out;
222

L
Linus Torvalds 已提交
223 224 225 226 227
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (!vma->vm_file)
			continue;
		if (!(vma->vm_flags & VM_EXECUTABLE))
			continue;
228
		cookie = fast_get_dcookie(&vma->vm_file->f_path);
L
Linus Torvalds 已提交
229 230 231 232 233 234 235 236 237 238 239 240 241
		break;
	}

out:
	return cookie;
}


/* Convert the EIP value of a sample into a persistent dentry/offset
 * pair that can then be added to the global event buffer. We make
 * sure to do this lookup before a mm->mmap modification happens so
 * we don't lose track.
 */
242 243
static unsigned long
lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset)
L
Linus Torvalds 已提交
244
{
245
	unsigned long cookie = NO_COOKIE;
246
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
247 248

	for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
249

L
Linus Torvalds 已提交
250 251 252
		if (addr < vma->vm_start || addr >= vma->vm_end)
			continue;

253
		if (vma->vm_file) {
254
			cookie = fast_get_dcookie(&vma->vm_file->f_path);
255 256 257 258 259 260 261
			*offset = (vma->vm_pgoff << PAGE_SHIFT) + addr -
				vma->vm_start;
		} else {
			/* must be an anonymous map */
			*offset = addr;
		}

L
Linus Torvalds 已提交
262 263 264
		break;
	}

265 266 267
	if (!vma)
		cookie = INVALID_COOKIE;

L
Linus Torvalds 已提交
268 269 270
	return cookie;
}

271
static unsigned long last_cookie = INVALID_COOKIE;
272

L
Linus Torvalds 已提交
273 274 275 276 277
static void add_cpu_switch(int i)
{
	add_event_entry(ESCAPE_CODE);
	add_event_entry(CPU_SWITCH_CODE);
	add_event_entry(i);
278
	last_cookie = INVALID_COOKIE;
L
Linus Torvalds 已提交
279 280 281 282 283 284
}

static void add_kernel_ctx_switch(unsigned int in_kernel)
{
	add_event_entry(ESCAPE_CODE);
	if (in_kernel)
285
		add_event_entry(KERNEL_ENTER_SWITCH_CODE);
L
Linus Torvalds 已提交
286
	else
287
		add_event_entry(KERNEL_EXIT_SWITCH_CODE);
L
Linus Torvalds 已提交
288
}
289

L
Linus Torvalds 已提交
290
static void
291
add_user_ctx_switch(struct task_struct const *task, unsigned long cookie)
L
Linus Torvalds 已提交
292 293
{
	add_event_entry(ESCAPE_CODE);
294
	add_event_entry(CTX_SWITCH_CODE);
L
Linus Torvalds 已提交
295 296 297 298 299 300 301 302
	add_event_entry(task->pid);
	add_event_entry(cookie);
	/* Another code for daemon back-compat */
	add_event_entry(ESCAPE_CODE);
	add_event_entry(CTX_TGID_CODE);
	add_event_entry(task->tgid);
}

303

L
Linus Torvalds 已提交
304 305 306 307 308 309 310
static void add_cookie_switch(unsigned long cookie)
{
	add_event_entry(ESCAPE_CODE);
	add_event_entry(COOKIE_SWITCH_CODE);
	add_event_entry(cookie);
}

311

L
Linus Torvalds 已提交
312 313 314 315 316 317
static void add_trace_begin(void)
{
	add_event_entry(ESCAPE_CODE);
	add_event_entry(TRACE_BEGIN_CODE);
}

318 319
#ifdef CONFIG_OPROFILE_IBS

320 321 322 323 324 325
#define IBS_FETCH_CODE_SIZE	2
#define IBS_OP_CODE_SIZE	5

/*
 * Add IBS fetch and op entries to event buffer
 */
326
static void add_ibs_begin(int cpu, int code, struct mm_struct *mm)
327 328 329 330 331
{
	unsigned long rip;
	int i, count;
	unsigned long ibs_cookie = 0;
	off_t offset;
332
	struct op_sample *sample;
333

334
	sample = op_cpu_buffer_read_entry(cpu);
335 336 337
	if (!sample)
		goto Error;
	rip = sample->eip;
338 339

#ifdef __LP64__
340
	rip += sample->event << 32;
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
#endif

	if (mm) {
		ibs_cookie = lookup_dcookie(mm, rip, &offset);

		if (ibs_cookie == NO_COOKIE)
			offset = rip;
		if (ibs_cookie == INVALID_COOKIE) {
			atomic_inc(&oprofile_stats.sample_lost_no_mapping);
			offset = rip;
		}
		if (ibs_cookie != last_cookie) {
			add_cookie_switch(ibs_cookie);
			last_cookie = ibs_cookie;
		}
	} else
		offset = rip;

	add_event_entry(ESCAPE_CODE);
	add_event_entry(code);
	add_event_entry(offset);	/* Offset from Dcookie */

	/* we send the Dcookie offset, but send the raw Linear Add also*/
364 365
	add_event_entry(sample->eip);
	add_event_entry(sample->event);
366 367 368 369 370 371 372

	if (code == IBS_FETCH_CODE)
		count = IBS_FETCH_CODE_SIZE;	/*IBS FETCH is 2 int64s*/
	else
		count = IBS_OP_CODE_SIZE;	/*IBS OP is 5 int64s*/

	for (i = 0; i < count; i++) {
373
		sample = op_cpu_buffer_read_entry(cpu);
374 375 376 377
		if (!sample)
			goto Error;
		add_event_entry(sample->eip);
		add_event_entry(sample->event);
378
	}
379 380 381 382 383

	return;

Error:
	return;
384
}
L
Linus Torvalds 已提交
385

386 387
#endif

L
Linus Torvalds 已提交
388 389 390 391 392 393 394
static void add_sample_entry(unsigned long offset, unsigned long event)
{
	add_event_entry(offset);
	add_event_entry(event);
}


R
Robert Richter 已提交
395 396 397 398 399 400 401
/*
 * Add a sample to the global event buffer. If possible the
 * sample is converted into a persistent dentry/offset pair
 * for later lookup from userspace. Return 0 on failure.
 */
static int
add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel)
L
Linus Torvalds 已提交
402 403 404
{
	unsigned long cookie;
	off_t offset;
405

R
Robert Richter 已提交
406 407 408 409 410 411 412 413 414 415 416 417
	if (in_kernel) {
		add_sample_entry(s->eip, s->event);
		return 1;
	}

	/* add userspace sample */

	if (!mm) {
		atomic_inc(&oprofile_stats.sample_lost_no_mm);
		return 0;
	}

418 419
	cookie = lookup_dcookie(mm, s->eip, &offset);

420
	if (cookie == INVALID_COOKIE) {
L
Linus Torvalds 已提交
421 422 423 424 425 426 427 428 429 430 431 432 433 434
		atomic_inc(&oprofile_stats.sample_lost_no_mapping);
		return 0;
	}

	if (cookie != last_cookie) {
		add_cookie_switch(cookie);
		last_cookie = cookie;
	}

	add_sample_entry(offset, s->event);

	return 1;
}

435 436

static void release_mm(struct mm_struct *mm)
L
Linus Torvalds 已提交
437 438 439 440 441 442 443 444
{
	if (!mm)
		return;
	up_read(&mm->mmap_sem);
	mmput(mm);
}


445
static struct mm_struct *take_tasks_mm(struct task_struct *task)
L
Linus Torvalds 已提交
446
{
447
	struct mm_struct *mm = get_task_mm(task);
L
Linus Torvalds 已提交
448 449 450 451 452 453 454 455 456 457
	if (mm)
		down_read(&mm->mmap_sem);
	return mm;
}


static inline int is_code(unsigned long val)
{
	return val == ESCAPE_CODE;
}
458

L
Linus Torvalds 已提交
459 460 461 462 463 464 465 466 467

/* Move tasks along towards death. Any tasks on dead_tasks
 * will definitely have no remaining references in any
 * CPU buffers at this point, because we use two lists,
 * and to have reached the list, it must have gone through
 * one full sync already.
 */
static void process_task_mortuary(void)
{
468 469
	unsigned long flags;
	LIST_HEAD(local_dead_tasks);
470 471
	struct task_struct *task;
	struct task_struct *ttask;
L
Linus Torvalds 已提交
472

473
	spin_lock_irqsave(&task_mortuary, flags);
L
Linus Torvalds 已提交
474

475 476
	list_splice_init(&dead_tasks, &local_dead_tasks);
	list_splice_init(&dying_tasks, &dead_tasks);
L
Linus Torvalds 已提交
477

478 479 480
	spin_unlock_irqrestore(&task_mortuary, flags);

	list_for_each_entry_safe(task, ttask, &local_dead_tasks, tasks) {
L
Linus Torvalds 已提交
481
		list_del(&task->tasks);
482
		free_task(task);
L
Linus Torvalds 已提交
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
	}
}


static void mark_done(int cpu)
{
	int i;

	cpu_set(cpu, marked_cpus);

	for_each_online_cpu(i) {
		if (!cpu_isset(i, marked_cpus))
			return;
	}

	/* All CPUs have been processed at least once,
	 * we can process the mortuary once
	 */
	process_task_mortuary();

	cpus_clear(marked_cpus);
}


/* FIXME: this is not sufficient if we implement syscall barrier backtrace
 * traversal, the code switch to sb_sample_start at first kernel enter/exit
 * switch so we need a fifth state and some special handling in sync_buffer()
 */
typedef enum {
	sb_bt_ignore = -2,
	sb_buffer_start,
	sb_bt_start,
	sb_sample_start,
} sync_buffer_state;

/* Sync one of the CPU's buffers into the global event buffer.
 * Here we need to go through each batch of samples punctuated
 * by context switch notes, taking the task's mmap_sem and doing
 * lookup in task->mm->mmap to convert EIP into dcookie/offset
 * value.
 */
void sync_buffer(int cpu)
{
	struct mm_struct *mm = NULL;
527
	struct mm_struct *oldmm;
528
	struct task_struct *new;
L
Linus Torvalds 已提交
529 530 531
	unsigned long cookie = 0;
	int in_kernel = 1;
	sync_buffer_state state = sb_buffer_start;
532
	unsigned int i;
L
Linus Torvalds 已提交
533 534
	unsigned long available;

535
	mutex_lock(&buffer_mutex);
536

L
Linus Torvalds 已提交
537 538
	add_cpu_switch(cpu);

539 540
	op_cpu_buffer_reset(cpu);
	available = op_cpu_buffer_entries(cpu);
L
Linus Torvalds 已提交
541 542

	for (i = 0; i < available; ++i) {
543
		struct op_sample *s = op_cpu_buffer_read_entry(cpu);
544 545
		if (!s)
			break;
546

L
Linus Torvalds 已提交
547
		if (is_code(s->eip)) {
548 549 550
			switch (s->event) {
			case 0:
			case CPU_IS_KERNEL:
L
Linus Torvalds 已提交
551 552 553 554 555
				/* kernel/userspace switch */
				in_kernel = s->event;
				if (state == sb_buffer_start)
					state = sb_sample_start;
				add_kernel_ctx_switch(s->event);
556 557
				break;
			case CPU_TRACE_BEGIN:
L
Linus Torvalds 已提交
558 559
				state = sb_bt_start;
				add_trace_begin();
560
				break;
561
#ifdef CONFIG_OPROFILE_IBS
562
			case IBS_FETCH_BEGIN:
563
				state = sb_bt_start;
564
				add_ibs_begin(cpu, IBS_FETCH_CODE, mm);
565 566
				break;
			case IBS_OP_BEGIN:
567
				state = sb_bt_start;
568
				add_ibs_begin(cpu, IBS_OP_CODE, mm);
569
				break;
570
#endif
571
			default:
L
Linus Torvalds 已提交
572
				/* userspace context switch */
573
				oldmm = mm;
L
Linus Torvalds 已提交
574 575 576 577 578 579
				new = (struct task_struct *)s->event;
				release_mm(oldmm);
				mm = take_tasks_mm(new);
				if (mm != oldmm)
					cookie = get_exec_dcookie(mm);
				add_user_ctx_switch(new, cookie);
580
				break;
L
Linus Torvalds 已提交
581
			}
582 583 584 585 586
		} else if (state >= sb_bt_start &&
			   !add_sample(mm, s, in_kernel)) {
			if (state == sb_bt_start) {
				state = sb_bt_ignore;
				atomic_inc(&oprofile_stats.bt_lost_no_mapping);
L
Linus Torvalds 已提交
587 588 589 590 591 592 593
			}
		}
	}
	release_mm(mm);

	mark_done(cpu);

594
	mutex_unlock(&buffer_mutex);
L
Linus Torvalds 已提交
595
}
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619

/* The function can be used to add a buffer worth of data directly to
 * the kernel buffer. The buffer is assumed to be a circular buffer.
 * Take the entries from index start and end at index end, wrapping
 * at max_entries.
 */
void oprofile_put_buff(unsigned long *buf, unsigned int start,
		       unsigned int stop, unsigned int max)
{
	int i;

	i = start;

	mutex_lock(&buffer_mutex);
	while (i != stop) {
		add_event_entry(buf[i++]);

		if (i >= max)
			i = 0;
	}

	mutex_unlock(&buffer_mutex);
}